time=2026-10-24T00:11:13.169Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-24T00:11:14.722Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-24T00:11:14.722Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-24T00:11:15.608Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-24T00:11:15.608Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:11:15.609Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j2212352-svc
time=2026-10-24T00:13:20.091Z level=INFO msg="Cluster provisioning state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded
time=2026-10-24T00:13:20.091Z level=INFO msg="ARM operation complete" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m4s
time=2026-10-24T00:13:20.091Z level=INFO msg="Step 1/9: Ensuring MISE ext-authz ConfigMap" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:20.407Z level=INFO msg="ConfigMap created" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30
time=2026-10-24T00:13:20.407Z level=INFO msg="Step 2/9: Checking tag-based namespace safety" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:20.407Z level=INFO msg="Step 3/9: Verifying control plane health and flipping tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:20.862Z level=INFO msg="Created revision tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30
time=2026-10-24T00:13:20.862Z level=INFO msg="Step 4/9: Ensuring ingress gateway annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:20.894Z level=INFO msg="Applying annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external
time=2026-10-24T00:13:21.097Z level=INFO msg="Step 5/9: Migrating workloads to target revision" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:21.168Z level=INFO msg="Step 6/9: Running health check" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:21.740Z level=INFO msg="Health check passed -- checking for orphaned workloads before completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:21.740Z level=INFO msg="Step 7/9: Running orphan guard" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:21.740Z level=INFO msg="No orphaned workloads -- completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:21.740Z level=INFO msg="Step 8/9: Completing canary via ARM (point of no return)" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-24T00:13:21.740Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j2212352-svc
time=2026-10-24T00:28:24.267Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"

time=2026-10-24T00:10:57.890Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description="Step deploy\n Kind: Helm\n"
time=2026-10-24T00:10:58.200Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-24T00:10:58.200Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-24T00:10:58.456Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller
time=2026-10-24T00:10:58.679Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false
time=2026-10-24T00:10:58.679Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-24T00:10:58.736Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-24T00:16:02.844Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-24T00:16:02.844Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x26e13640a140>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-jft4smjjlxxw" to finish creating: GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220",
      "name": "fb71e438-88d3-4b63-82a0-b820e717c220",
      "status": "Failed",
      "startTime": "2026-10-24T00:13:46.41509851Z",
      "endTime": "2026-10-24T00:32:46.057566699Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220
            correlation ID: 97b55a4e-bfeb-47d6-99bd-fc35a2cc6359
            client request ID: c7f67463-49ac-4db2-9fc4-28f026c30acf
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x26e13640a140>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-jft4smjjlxxw" to finish creating: GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220",
      "name": "fb71e438-88d3-4b63-82a0-b820e717c220",
      "status": "Failed",
      "startTime": "2026-10-24T00:13:46.41509851Z",
      "endTime": "2026-10-24T00:32:46.057566699Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220
            correlation ID: 97b55a4e-bfeb-47d6-99bd-fc35a2cc6359
            client request ID: c7f67463-49ac-4db2-9fc4-28f026c30acf

time=2026-10-23T22:20:23.920Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-23T22:20:25.642Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-23T22:20:25.642Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-23T22:20:26.982Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-23T22:20:26.982Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:20:26.982Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j9780224-svc
time=2026-10-23T22:23:03.498Z level=INFO msg="Cluster provisioning state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded
time=2026-10-23T22:23:03.499Z level=INFO msg="ARM operation complete" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m37s
time=2026-10-23T22:23:03.499Z level=INFO msg="Step 1/9: Ensuring MISE ext-authz ConfigMap" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:04.054Z level=INFO msg="ConfigMap created" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30
time=2026-10-23T22:23:04.054Z level=INFO msg="Step 2/9: Checking tag-based namespace safety" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:04.054Z level=INFO msg="Step 3/9: Verifying control plane health and flipping tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:04.811Z level=INFO msg="Created revision tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30
time=2026-10-23T22:23:04.811Z level=INFO msg="Step 4/9: Ensuring ingress gateway annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:04.874Z level=INFO msg="Applying annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external
time=2026-10-23T22:23:05.250Z level=INFO msg="Step 5/9: Migrating workloads to target revision" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:05.375Z level=INFO msg="Step 6/9: Running health check" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:06.263Z level=INFO msg="Health check passed -- checking for orphaned workloads before completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:06.263Z level=INFO msg="Step 7/9: Running orphan guard" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:06.263Z level=INFO msg="No orphaned workloads -- completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:06.263Z level=INFO msg="Step 8/9: Completing canary via ARM (point of no return)" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T22:23:06.263Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j9780224-svc
time=2026-10-23T22:38:09.485Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x35e0b0894420>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-9pzwhwthgd8h" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df",
      "name": "4735d84e-eeda-4991-9a00-15bdfb0112df",
      "status": "Failed",
      "startTime": "2026-10-23T22:59:37.48096278Z",
      "endTime": "2026-10-23T23:18:39.961096304Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df
            correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e
            client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x35e0b0894420>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-9pzwhwthgd8h" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df",
      "name": "4735d84e-eeda-4991-9a00-15bdfb0112df",
      "status": "Failed",
      "startTime": "2026-10-23T22:59:37.48096278Z",
      "endTime": "2026-10-23T23:18:39.961096304Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df
            correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e
            client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.
ClusterIssuer CRD should be installed and the resource creatable
Expected success, but got an error:
    <*errors.StatusError | 0x157c9dc7bc20>: 
    Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s": no endpoints available for service "cert-manager-webhook"
    {
        ErrStatus: {
            TypeMeta: {Kind: "Status", APIVersion: "v1"},
            ListMeta: {
                SelfLink: "",
                ResourceVersion: "",
                Continue: "",
                RemainingItemCount: nil,
                ShardInfo: nil,
            },
            Status: "Failure",
            Message: "Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
            Reason: "InternalError",
            Details: {
                Name: "",
                Group: "",
                Kind: "",
                UID: "",
                Causes: [
                    {
                        Type: "",
                        Message: "failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
                        Field: "",
                    },
                ],
                RetryAfterSeconds: 0,
            },
            Code: 500,
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.
ClusterIssuer CRD should be installed and the resource creatable
Expected success, but got an error:
    <*errors.StatusError | 0x157c9dc7bc20>: 
    Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s": no endpoints available for service "cert-manager-webhook"
    {
        ErrStatus: {
            TypeMeta: {Kind: "Status", APIVersion: "v1"},
            ListMeta: {
                SelfLink: "",
                ResourceVersion: "",
                Continue: "",
                RemainingItemCount: nil,
                ShardInfo: nil,
            },
            Status: "Failure",
            Message: "Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
            Reason: "InternalError",
            Details: {
                Name: "",
                Group: "",
                Kind: "",
                UID: "",
                Causes: [
                    {
                        Type: "",
                        Message: "failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
                        Field: "",
                    },
                ],
                RetryAfterSeconds: 0,
            },
            Code: 500,
        },
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x348180f2e220>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*verifiers.pollObservationError | 0x348180f2e200>{
            observation: <*errors.errorString | 0x3481807fdb50>{
                s: "dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
            },
            interruption: <*fmt.wrapError | 0x348180f2e1c0>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x3481808801e0>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x348180f2e1a0>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x3481828e4120>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x348180f2e220>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*verifiers.pollObservationError | 0x348180f2e200>{
            observation: <*errors.errorString | 0x3481807fdb50>{
                s: "dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
            },
            interruption: <*fmt.wrapError | 0x348180f2e1c0>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x3481808801e0>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x348180f2e1a0>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x3481828e4120>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-23T22:38:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9171712-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.104:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-cqcsl", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="24c5a63c-ba67-481c-97bd-1b2284e4616c"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-cqcsl (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-23T22:40:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9171712-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.104:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-cqcsl", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="24c5a63c-ba67-481c-97bd-1b2284e4616c"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-cqcsl (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-23T22:41:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9171712-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.221:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-l6h2t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="dc5bdfb6-fff6-4941-9c02-12766a972bae"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-l6h2t (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-23T22:43:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9171712-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.221:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-l6h2t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="dc5bdfb6-fff6-4941-9c02-12766a972bae"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-l6h2t (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xac144fee4a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0xac1449cf1d0>{
            s: "dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xac144fee4a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0xac1449cf1d0>{
            s: "dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x1355449513d0>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451->172.30.0.10:53: read: connection refused",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x1355449513d0>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451->172.30.0.10:53: read: connection refused",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:328]: failed to create worker nodepool for serial console test
Unexpected error:
    <*fmt.wrapError | 0x204486f8560>: 
    failed to create NodePool worker: failed waiting for nodepool="worker" for cluster "sre-hcp-cluster-sc" in resourcegroup="admin-api-serialconsole-wrxzvkddb94j" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5",
      "name": "13760d44-2d42-4c53-9fca-8d97381d23a5",
      "status": "Failed",
      "startTime": "2026-10-23T21:19:36.261324147Z",
      "endTime": "2026-10-23T21:38:45.545540127Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5
            correlation ID: 5d55a2cb-56aa-47f7-a5b9-cf7b9a27d226
            client request ID: f043a8e7-f2d1-473c-b2e3-11abda25efb3
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:328]: failed to create worker nodepool for serial console test
Unexpected error:
    <*fmt.wrapError | 0x204486f8560>: 
    failed to create NodePool worker: failed waiting for nodepool="worker" for cluster "sre-hcp-cluster-sc" in resourcegroup="admin-api-serialconsole-wrxzvkddb94j" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5",
      "name": "13760d44-2d42-4c53-9fca-8d97381d23a5",
      "status": "Failed",
      "startTime": "2026-10-23T21:19:36.261324147Z",
      "endTime": "2026-10-23T21:38:45.545540127Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5
            correlation ID: 5d55a2cb-56aa-47f7-a5b9-cf7b9a27d226
            client request ID: f043a8e7-f2d1-473c-b2e3-11abda25efb3
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x3dd71ecc7860>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-gvwnv545c9xf" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c",
      "name": "b77ed110-707e-4c38-ab0a-33ad172ee68c",
      "status": "Failed",
      "startTime": "2026-10-23T21:19:37.486805843Z",
      "endTime": "2026-10-23T21:38:45.647369611Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c
            correlation ID: 6572f7b9-3e5a-422f-aa65-926c920bbad1
            client request ID: bb39b90a-4ada-4996-91b9-6bd38bf61535
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x3dd71ecc7860>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-gvwnv545c9xf" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c",
      "name": "b77ed110-707e-4c38-ab0a-33ad172ee68c",
      "status": "Failed",
      "startTime": "2026-10-23T21:19:37.486805843Z",
      "endTime": "2026-10-23T21:38:45.647369611Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c
            correlation ID: 6572f7b9-3e5a-422f-aa65-926c920bbad1
            client request ID: bb39b90a-4ada-4996-91b9-6bd38bf61535
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0xa03dff15100>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-k98p95t7rwlk" to finish creating: GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8",
      "name": "1937a216-e9d1-46bb-b9f8-a55a34e91da8",
      "status": "Failed",
      "startTime": "2026-10-23T20:32:20.327838457Z",
      "endTime": "2026-10-23T20:51:29.045056606Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-2p298-8vgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-2p298-8vgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8
            correlation ID: f3bdc38e-f649-43a0-824c-a60a987dceb4
            client request ID: f8924ec9-c563-43bf-b506-67458e3fbd1f
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0xa03dff15100>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-k98p95t7rwlk" to finish creating: GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8",
      "name": "1937a216-e9d1-46bb-b9f8-a55a34e91da8",
      "status": "Failed",
      "startTime": "2026-10-23T20:32:20.327838457Z",
      "endTime": "2026-10-23T20:51:29.045056606Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-2p298-8vgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-2p298-8vgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8
            correlation ID: f3bdc38e-f649-43a0-824c-a60a987dceb4
            client request ID: f8924ec9-c563-43bf-b506-67458e3fbd1f

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster "cluster-hs-pre-7lnp9z" should complete its version rollout
Unexpected error:
    <*fmt.wrapError | 0x1290b3e7a900>: 
    VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39="Partial"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available
    {
        msg: "VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
        err: <*verifiers.pollObservationError | 0x1290b3e7a840>{
            observation: <*errors.errorString | 0x1290b436a5d0>{
                s: "clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
            },
            interruption: <*fmt.wrapError | 0x1290b3e7a820>{
                msg: "failed to get clusterversion \"version\": context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster "cluster-hs-pre-7lnp9z" should complete its version rollout
Unexpected error:
    <*fmt.wrapError | 0x1290b3e7a900>: 
    VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39="Partial"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available
    {
        msg: "VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
        err: <*verifiers.pollObservationError | 0x1290b3e7a840>{
            observation: <*errors.errorString | 0x1290b436a5d0>{
                s: "clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
            },
            interruption: <*fmt.wrapError | 0x1290b3e7a820>{
                msg: "failed to get clusterversion \"version\": context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*errors.errorString | 0x2b8958bd95b0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*errors.errorString | 0x2b8958bd95b0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-hbhgzqp5zds5/cluster-candidate-4-22-9t7f9b should provision
Unexpected error:
    <*fmt.wrapError | 0x2655fb8fc880>: 
    failed to create HCP cluster cluster-candidate-4-22-9t7f9b: failed waiting for cluster="cluster-candidate-4-22-9t7f9b" in resourcegroup="rg-candidate-4-22-hbhgzqp5zds5" to finish creating: GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d",
      "name": "9cab47e8-7ab4-4765-9ac8-c0235f97fa6d",
      "status": "Failed",
      "startTime": "2026-10-23T19:29:36.620172443Z",
      "endTime": "2026-10-23T19:58:43.098929016Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d
            correlation ID: 91833dfb-2346-485c-9abf-90635cc143aa
            client request ID: 8cd6038e-c1e2-4db7-aaf4-36204667b8f5
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-hbhgzqp5zds5/cluster-candidate-4-22-9t7f9b should provision
Unexpected error:
    <*fmt.wrapError | 0x2655fb8fc880>: 
    failed to create HCP cluster cluster-candidate-4-22-9t7f9b: failed waiting for cluster="cluster-candidate-4-22-9t7f9b" in resourcegroup="rg-candidate-4-22-hbhgzqp5zds5" to finish creating: GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d",
      "name": "9cab47e8-7ab4-4765-9ac8-c0235f97fa6d",
      "status": "Failed",
      "startTime": "2026-10-23T19:29:36.620172443Z",
      "endTime": "2026-10-23T19:58:43.098929016Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d
            correlation ID: 91833dfb-2346-485c-9abf-90635cc143aa
            client request ID: 8cd6038e-c1e2-4db7-aaf4-36204667b8f5
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T19:51:45Z
  Ended:       2026-10-23T20:15:45Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd has not matched the expected number of replicas for longer than 15 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-23T19:51:45Z
  Ended:       2026-10-23T20:15:45Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd has not matched the expected number of replicas for longer than 15 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T19:53:47Z
  Ended:       2026-10-23T20:19:41Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="ad470ce0-381a-4372-b725-f995a508a24f"
  Description: Pod ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-23T19:53:47Z
  Ended:       2026-10-23T20:19:41Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="ad470ce0-381a-4372-b725-f995a508a24f"
  Description: Pod ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T20:05:44Z
  Ended:       2026-10-23T20:17:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-23T20:05:45Z
  Ended:       2026-10-23T20:17:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-23T19:45:26Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-wpx95", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f99f4808-64ca-4f14-b866-30cd9d5a44aa"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-wpx95 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-23T19:48:26Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j5932672-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.87:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-b4nrw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="b83bb111-3a9c-4b96-b2ca-76053771e1a5"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-b4nrw (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-23T19:48:28Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j5932672-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.87:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-b4nrw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="b83bb111-3a9c-4b96-b2ca-76053771e1a5"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-b4nrw (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-23T19:51:26Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j5932672-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.248:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-wpx95", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f99f4808-64ca-4f14-b866-30cd9d5a44aa"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-wpx95 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster "cluster-candidate-4-23-dzr6vp"
Unexpected error:
    <*errors.joinError | 0x3fde2a0147b0>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm": nodes "p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0x3fde2a4d4080>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0x3fde2a4d4040>{
                    msg: "failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0x3fde2a4d4020>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0x3fde2aa98140>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster "cluster-candidate-4-23-dzr6vp"
Unexpected error:
    <*errors.joinError | 0x3fde2a0147b0>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm": nodes "p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0x3fde2a4d4080>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0x3fde2a4d4040>{
                    msg: "failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0x3fde2a4d4020>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0x3fde2aa98140>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred

time=2026-10-23T18:08:55.483Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-23T18:08:56.785Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-23T18:08:56.785Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-23T18:08:57.580Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-23T18:08:57.580Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:08:57.580Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j6973824-svc
time=2026-10-23T18:13:02.175Z level=INFO msg="Cluster provisioning state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded
time=2026-10-23T18:13:02.175Z level=INFO msg="ARM operation complete" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=4m5s
time=2026-10-23T18:13:02.175Z level=INFO msg="Step 1/9: Ensuring MISE ext-authz ConfigMap" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:02.795Z level=INFO msg="ConfigMap created" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30
time=2026-10-23T18:13:02.795Z level=INFO msg="Step 2/9: Checking tag-based namespace safety" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:02.795Z level=INFO msg="Step 3/9: Verifying control plane health and flipping tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:03.759Z level=INFO msg="Created revision tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30
time=2026-10-23T18:13:03.759Z level=INFO msg="Step 4/9: Ensuring ingress gateway annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:03.939Z level=INFO msg="Applying annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external
time=2026-10-23T18:13:04.167Z level=INFO msg="Step 5/9: Migrating workloads to target revision" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:04.333Z level=INFO msg="Step 6/9: Running health check" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:04.935Z level=INFO msg="Health check passed -- checking for orphaned workloads before completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:04.935Z level=INFO msg="Step 7/9: Running orphan guard" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:04.935Z level=INFO msg="No orphaned workloads -- completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:04.935Z level=INFO msg="Step 8/9: Completing canary via ARM (point of no return)" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T18:13:04.935Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j6973824-svc
time=2026-10-23T18:28:07.753Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool "np-sn4xdl" for cluster "cluster-candidate-4-21-sn4xdl"
Unexpected error:
    <*fmt.wrapError | 0x60436518ee0>: 
    failed to create NodePool np-sn4xdl: failed waiting for nodepool="np-sn4xdl" for cluster "cluster-candidate-4-21-sn4xdl" in resourcegroup="rg-candidate-4-21-c4tsdrwbnnfb" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e",
      "name": "d83a0f27-909b-4319-9bb2-43e65d178f6e",
      "status": "Failed",
      "startTime": "2026-10-23T18:34:43.074031046Z",
      "endTime": "2026-10-23T18:53:43.987521771Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e
            correlation ID: 327dae19-27b0-4a9b-8e18-5ab3841e0ec7
            client request ID: 2803ecfb-10a6-4276-a6a2-935c10a7ca77
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool "np-sn4xdl" for cluster "cluster-candidate-4-21-sn4xdl"
Unexpected error:
    <*fmt.wrapError | 0x60436518ee0>: 
    failed to create NodePool np-sn4xdl: failed waiting for nodepool="np-sn4xdl" for cluster "cluster-candidate-4-21-sn4xdl" in resourcegroup="rg-candidate-4-21-c4tsdrwbnnfb" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e",
      "name": "d83a0f27-909b-4319-9bb2-43e65d178f6e",
      "status": "Failed",
      "startTime": "2026-10-23T18:34:43.074031046Z",
      "endTime": "2026-10-23T18:53:43.987521771Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e
            correlation ID: 327dae19-27b0-4a9b-8e18-5ab3841e0ec7
            client request ID: 2803ecfb-10a6-4276-a6a2-935c10a7ca77
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x21c5dc2ca7e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-f88rfw7wtj4v" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5",
      "name": "b02b5a9b-3f7f-467b-bc65-f56966e21cc5",
      "status": "Failed",
      "startTime": "2026-10-23T18:32:08.161782793Z",
      "endTime": "2026-10-23T18:51:14.083013713Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f88rfw7wtj4v--managed/ingress-cert-le-np-1-b4g5h-9bn6x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f88rfw7wtj4v--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-b4g5h-9bn6x-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5
            correlation ID: 053e3b77-21ef-4381-b997-ce492e16aa00
            client request ID: d87da6eb-f6e0-450c-b35a-d8d6ce034e64
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x21c5dc2ca7e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-f88rfw7wtj4v" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5",
      "name": "b02b5a9b-3f7f-467b-bc65-f56966e21cc5",
      "status": "Failed",
      "startTime": "2026-10-23T18:32:08.161782793Z",
      "endTime": "2026-10-23T18:51:14.083013713Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f88rfw7wtj4v--managed/ingress-cert-le-np-1-b4g5h-9bn6x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f88rfw7wtj4v--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-b4g5h-9bn6x-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5
            correlation ID: 053e3b77-21ef-4381-b997-ce492e16aa00
            client request ID: d87da6eb-f6e0-450c-b35a-d8d6ce034e64

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster "sre-hcp-cluster"
Unexpected error:
    <*fmt.wrapError | 0x10298a136780>: 
    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster="sre-hcp-cluster" in resourcegroup="admin-api-breakglass-thv2btpwv2x8" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13",
      "name": "505bd364-3947-46d9-aabe-9bb9ef0e8d13",
      "status": "Failed",
      "startTime": "2026-10-23T18:20:41.038643697Z",
      "endTime": "2026-10-23T18:49:43.826799473Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 9m9s ago"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13
            correlation ID: fb84c414-e895-47e3-bdef-c718ca7da71b
            client request ID: e0f67746-2722-4f93-b4a1-3c122c4d2e69
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster "sre-hcp-cluster"
Unexpected error:
    <*fmt.wrapError | 0x10298a136780>: 
    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster="sre-hcp-cluster" in resourcegroup="admin-api-breakglass-thv2btpwv2x8" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13",
      "name": "505bd364-3947-46d9-aabe-9bb9ef0e8d13",
      "status": "Failed",
      "startTime": "2026-10-23T18:20:41.038643697Z",
      "endTime": "2026-10-23T18:49:43.826799473Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 9m9s ago"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13
            correlation ID: fb84c414-e895-47e3-bdef-c718ca7da71b
            client request ID: e0f67746-2722-4f93-b4a1-3c122c4d2e69
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster "sre-hcp-cluster-sc" for serial console test
Unexpected error:
    <*fmt.wrapError | 0x29d3447c6480>: 
    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster="sre-hcp-cluster-sc" in resourcegroup="admin-api-serialconsole-gbjmnzzdfntv" to finish creating: GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae",
      "name": "644d90e9-61e3-42ef-a2dd-083bda41c8ae",
      "status": "Failed",
      "startTime": "2026-10-23T17:53:58.718650875Z",
      "endTime": "2026-10-23T18:23:02.815422763Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae
            correlation ID: 6d68ce4a-22a8-4eea-a977-b246a66b35ea
            client request ID: 9a628c19-f8fd-4bf6-859e-bb2f0567065e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster "sre-hcp-cluster-sc" for serial console test
Unexpected error:
    <*fmt.wrapError | 0x29d3447c6480>: 
    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster="sre-hcp-cluster-sc" in resourcegroup="admin-api-serialconsole-gbjmnzzdfntv" to finish creating: GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae",
      "name": "644d90e9-61e3-42ef-a2dd-083bda41c8ae",
      "status": "Failed",
      "startTime": "2026-10-23T17:53:58.718650875Z",
      "endTime": "2026-10-23T18:23:02.815422763Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae
            correlation ID: 6d68ce4a-22a8-4eea-a977-b246a66b35ea
            client request ID: 9a628c19-f8fd-4bf6-859e-bb2f0567065e
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T18:28:28Z
  Ended:       2026-10-23T18:40:28Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j8170368-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.219:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-23T18:28:28Z
  Ended:       2026-10-23T18:40:28Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j8170368-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.219:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T18:17:28Z
  Ended:       2026-10-23T18:44:27Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j8170368-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.219:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="9b5be082-3743-4901-bdab-8d87701f28f1"
  Description: Pod ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-23T18:17:28Z
  Ended:       2026-10-23T18:44:27Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j8170368-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.219:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="9b5be082-3743-4901-bdab-8d87701f28f1"
  Description: Pod ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T18:14:34Z
  Ended:       2026-10-23T18:39:29Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j8170368-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.219:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd has not matched the expected number of replicas for longer than 15 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-23T18:14:35Z
  Ended:       2026-10-23T18:39:29Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j8170368-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.219:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd has not matched the expected number of replicas for longer than 15 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster
Unexpected error:
    <*fmt.wrapError | 0xc108f3a63c0>: 
    failed to create customer-infra: failed waiting for deployment "customer-infra-private-ingress-9rjzw7" in resourcegroup="private-ingress-zqfzzkb8lcwr" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeploymentFailed
    --------------------------------------------------------------------------------
    {
      "status": "Failed",
      "error": {
        "code": "DeploymentFailed",
        "message": "At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.",
        "details": [
          {
            "code": "InternalServerError",
            "message": "{\r\n  \"error\": {\r\n    \"code\": \"ResourceDeploymentFailure\",\r\n    \"message\": \"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\"\r\n  }\r\n}"
          }
        ]
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365
            correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster
Unexpected error:
    <*fmt.wrapError | 0xc108f3a63c0>: 
    failed to create customer-infra: failed waiting for deployment "customer-infra-private-ingress-9rjzw7" in resourcegroup="private-ingress-zqfzzkb8lcwr" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeploymentFailed
    --------------------------------------------------------------------------------
    {
      "status": "Failed",
      "error": {
        "code": "DeploymentFailed",
        "message": "At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.",
        "details": [
          {
            "code": "InternalServerError",
            "message": "{\r\n  \"error\": {\r\n    \"code\": \"ResourceDeploymentFailure\",\r\n    \"message\": \"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\"\r\n  }\r\n}"
          }
        ]
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365
            correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0xa64ef1027e0>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-h54k9n8pjkjm" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882",
      "name": "1e0c8c9f-614e-479d-8628-81206456e882",
      "status": "Failed",
      "startTime": "2026-10-23T17:54:15.481914664Z",
      "endTime": "2026-10-23T18:23:16.346423782Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882
            correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77
            client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0xa64ef1027e0>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-h54k9n8pjkjm" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882",
      "name": "1e0c8c9f-614e-479d-8628-81206456e882",
      "status": "Failed",
      "startTime": "2026-10-23T17:54:15.481914664Z",
      "endTime": "2026-10-23T18:23:16.346423782Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882
            correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77
            client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x794496e4000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x79449964570>{
            s: "dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x794496e4000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x79449964570>{
            s: "dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred

alert fired 4 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T18:30:05Z
  Ended:       2026-10-23T18:41:05Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9103104-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.65.24:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-t567t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-23T18:30:05Z
  Ended:       2026-10-23T18:42:04Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9103104-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.65.24:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-t567t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-23T18:31:05Z
  Ended:       2026-10-23T18:40:13Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9103104-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.24:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-t567t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-23T18:31:05Z
  Ended:       2026-10-23T18:40:13Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9103104-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.24:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-t567t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x319d4a6beb60>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-f48bqq4pfgdf" to finish creating: GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468",
      "name": "96166412-c6d9-4ece-a45c-cf56e7487468",
      "status": "Failed",
      "startTime": "2026-10-23T16:53:56.577784124Z",
      "endTime": "2026-10-23T17:13:00.532320401Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-t2t6b-kptmf: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f48bqq4pfgdf--managed/ingress-cert-le-np-1-t2t6b-kptmf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f48bqq4pfgdf--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-t2t6b-kptmf-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available. Refer t... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468
            correlation ID: 1798d4dc-6289-4a4b-aaed-4eef33b20f02
            client request ID: 11e72bdf-c049-47a8-91d7-be1bc1c7965f
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x319d4a6beb60>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-f48bqq4pfgdf" to finish creating: GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468",
      "name": "96166412-c6d9-4ece-a45c-cf56e7487468",
      "status": "Failed",
      "startTime": "2026-10-23T16:53:56.577784124Z",
      "endTime": "2026-10-23T17:13:00.532320401Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-t2t6b-kptmf: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f48bqq4pfgdf--managed/ingress-cert-le-np-1-t2t6b-kptmf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f48bqq4pfgdf--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-t2t6b-kptmf-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '3' seconds after additional tokens are available. Refer t... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5718656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96166412-c6d9-4ece-a45c-cf56e7487468
            correlation ID: 1798d4dc-6289-4a4b-aaed-4eef33b20f02
            client request ID: 11e72bdf-c049-47a8-91d7-be1bc1c7965f

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed
Expected success, but got an error:
    <*errors.joinError | 0x1025b98de6a8>: 
    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get "https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        errs: [
            <*fmt.wrapError | 0x1025b95ed360>{
                msg: "VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                err: <*errors.joinError | 0x1025b9b81ec0>{
                    errs: [
                        <*errors.errorString | 0x1025b92c2ec0>{
                            s: "apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                        },
                    ],
                },
            },
        ],
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed
Expected success, but got an error:
    <*errors.joinError | 0x1025b98de6a8>: 
    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get "https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        errs: [
            <*fmt.wrapError | 0x1025b95ed360>{
                msg: "VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                err: <*errors.joinError | 0x1025b9b81ec0>{
                    errs: [
                        <*errors.errorString | 0x1025b92c2ec0>{
                            s: "apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                        },
                    ],
                },
            },
        ],
    }

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x19a5a6aee900>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-662w726xqd2k" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216",
      "name": "91414204-0657-405d-a101-7608067a7216",
      "status": "Failed",
      "startTime": "2026-10-23T15:54:03.184233444Z",
      "endTime": "2026-10-23T16:13:03.632601143Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216
            correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659
            client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x19a5a6aee900>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-662w726xqd2k" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216",
      "name": "91414204-0657-405d-a101-7608067a7216",
      "status": "Failed",
      "startTime": "2026-10-23T15:54:03.184233444Z",
      "endTime": "2026-10-23T16:13:03.632601143Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216
            correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659
            client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131
occurred

time=2026-10-23T15:33:31.685Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description="Step deploy\n Kind: Helm\n"
time=2026-10-23T15:33:32.582Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-23T15:33:32.582Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T15:33:33.039Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller
time=2026-10-23T15:33:33.396Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false
time=2026-10-23T15:33:33.396Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T15:33:33.530Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T15:38:39.422Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-23T15:38:39.423Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T15:27:17Z
  Ended:       2026-10-23T15:33:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5062656-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tkibdc5gs5nul5ksrupnsegrco0ubpn-w6p5u2c0q5q4w4r", pod="ignition-server-proxy-bf7d94666-n9xlf", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tkibdc5gs5nul5ksrupnsegrco0ubpn-w6p5u2c0q5q4w4r/ignition-server-proxy-bf7d94666-n9xlf has been in a non-ready state for longer than 5 minutes.

time=2026-10-23T14:27:54.359Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-23T14:27:55.884Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-23T14:27:55.885Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-23T14:27:56.758Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-23T14:27:56.759Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:27:56.759Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j5183488-svc
time=2026-10-23T14:30:02.581Z level=INFO msg="Cluster provisioning state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded
time=2026-10-23T14:30:02.581Z level=INFO msg="ARM operation complete" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m6s
time=2026-10-23T14:30:02.581Z level=INFO msg="Step 1/9: Ensuring MISE ext-authz ConfigMap" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:03.058Z level=INFO msg="ConfigMap created" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30
time=2026-10-23T14:30:03.058Z level=INFO msg="Step 2/9: Checking tag-based namespace safety" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:03.059Z level=INFO msg="Step 3/9: Verifying control plane health and flipping tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:03.941Z level=INFO msg="Created revision tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30
time=2026-10-23T14:30:03.941Z level=INFO msg="Step 4/9: Ensuring ingress gateway annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:03.972Z level=INFO msg="Applying annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external
time=2026-10-23T14:30:04.189Z level=INFO msg="Step 5/9: Migrating workloads to target revision" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:04.255Z level=INFO msg="Step 6/9: Running health check" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:04.808Z level=INFO msg="Health check passed -- checking for orphaned workloads before completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:04.808Z level=INFO msg="Step 7/9: Running orphan guard" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:04.808Z level=INFO msg="No orphaned workloads -- completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:04.808Z level=INFO msg="Step 8/9: Completing canary via ARM (point of no return)" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5183488-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T14:30:04.808Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j5183488-svc
time=2026-10-23T14:45:07.671Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"

alert fired 62 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-q7xh2", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-q7xh2 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-ndlbl", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-ndlbl has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-7xjfm", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-7xjfm has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-5dvf5", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-5dvf5 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-mb68l", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-mb68l has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-ssk4m", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-ssk4m has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-9ljfg", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-9ljfg has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-hlrms", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-hlrms has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-7zxmv", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-7zxmv has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-995kh", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-995kh has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-jj844", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-jj844 has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-hk2pc", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-hk2pc has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-v22fj", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-v22fj has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-x55nd", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-x55nd has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-dbxr7", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-dbxr7 has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-gmslj", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-gmslj has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-tfmkl", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-tfmkl has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-wcfrc", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-wcfrc has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-vz2cv", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-vz2cv has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-9mcmc", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-9mcmc has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-d6nhd", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-d6nhd has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-9txxf", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-9txxf has been in a non-ready state for longer than 5 minutes.
Firing 23:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-6b8wc", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-6b8wc has been in a non-ready state for longer than 5 minutes.
Firing 24:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-nrgxf", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-nrgxf has been in a non-ready state for longer than 5 minutes.
Firing 25:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-sgtgx", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-sgtgx has been in a non-ready state for longer than 5 minutes.
Firing 26:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-rqj2v", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-rqj2v has been in a non-ready state for longer than 5 minutes.
Firing 27:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-8vhqg", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-8vhqg has been in a non-ready state for longer than 5 minutes.
Firing 28:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-rsscm", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-rsscm has been in a non-ready state for longer than 5 minutes.
Firing 29:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-t4fln", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-t4fln has been in a non-ready state for longer than 5 minutes.
Firing 30:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-tjxjb", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-tjxjb has been in a non-ready state for longer than 5 minutes.
Firing 31:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-m5d5c", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-m5d5c has been in a non-ready state for longer than 5 minutes.
Firing 32:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-bhfw8", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-bhfw8 has been in a non-ready state for longer than 5 minutes.
Firing 33:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-8kc57", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-8kc57 has been in a non-ready state for longer than 5 minutes.
Firing 34:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-nmlxd", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-nmlxd has been in a non-ready state for longer than 5 minutes.
Firing 35:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-7nz9z", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-7nz9z has been in a non-ready state for longer than 5 minutes.
Firing 36:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-rr5tb", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-rr5tb has been in a non-ready state for longer than 5 minutes.
Firing 37:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-zgzmg", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-zgzmg has been in a non-ready state for longer than 5 minutes.
Firing 38:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-v2xpg", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-v2xpg has been in a non-ready state for longer than 5 minutes.
Firing 39:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-v96bl", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-v96bl has been in a non-ready state for longer than 5 minutes.
Firing 40:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-j9jl5", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-j9jl5 has been in a non-ready state for longer than 5 minutes.
Firing 41:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-b87xs", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-b87xs has been in a non-ready state for longer than 5 minutes.
Firing 42:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-pphfm", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-pphfm has been in a non-ready state for longer than 5 minutes.
Firing 43:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-hgkjq", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-hgkjq has been in a non-ready state for longer than 5 minutes.
Firing 44:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-xd72x", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-xd72x has been in a non-ready state for longer than 5 minutes.
Firing 45:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-8jcjf", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-8jcjf has been in a non-ready state for longer than 5 minutes.
Firing 46:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-rvdns", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-rvdns has been in a non-ready state for longer than 5 minutes.
Firing 47:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-x4kzj", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-x4kzj has been in a non-ready state for longer than 5 minutes.
Firing 48:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-b8qd2", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-b8qd2 has been in a non-ready state for longer than 5 minutes.
Firing 49:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-qt827", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-qt827 has been in a non-ready state for longer than 5 minutes.
Firing 50:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-q8mvg", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-q8mvg has been in a non-ready state for longer than 5 minutes.
Firing 51:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-kfjrt", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-kfjrt has been in a non-ready state for longer than 5 minutes.
Firing 52:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-hwkjz", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-hwkjz has been in a non-ready state for longer than 5 minutes.
Firing 53:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-9td26", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-9td26 has been in a non-ready state for longer than 5 minutes.
Firing 54:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-cr96z", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-cr96z has been in a non-ready state for longer than 5 minutes.
Firing 55:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-9dqr2", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-9dqr2 has been in a non-ready state for longer than 5 minutes.
Firing 56:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-d56c6", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-d56c6 has been in a non-ready state for longer than 5 minutes.
Firing 57:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-s89qq", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-s89qq has been in a non-ready state for longer than 5 minutes.
Firing 58:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-k2xxx", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-k2xxx has been in a non-ready state for longer than 5 minutes.
Firing 59:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-jb7dj", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-jb7dj has been in a non-ready state for longer than 5 minutes.
Firing 60:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-l6zq2", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-l6zq2 has been in a non-ready state for longer than 5 minutes.
Firing 61:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-hzddl", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-hzddl has been in a non-ready state for longer than 5 minutes.
Firing 62:
  State:       Fired (not resolved)
  Started:     2026-10-23T14:33:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5183488-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-k2wfc", region="canadacentral", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-k2wfc has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x3ce657fb6660>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-f9dtl496nd4k" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f",
      "name": "5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f",
      "status": "Failed",
      "startTime": "2026-10-23T13:52:15.708697202Z",
      "endTime": "2026-10-23T14:11:23.696759282Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-f9dtl496nd4k--managed/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-f9dtl496nd4k--managed/providers/Microsoft.Network/networkInterfaces/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to http... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f
            correlation ID: 2e16a443-f7c3-4087-8290-e874ac42ea22
            client request ID: d8c04265-7efa-4df2-aadc-80112544e6fc
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool "np-1" for fips-enabled cluster "fips-enabled-cluster"
Unexpected error:
    <*fmt.wrapError | 0x3ce657fb6660>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "fips-enabled-cluster" in resourcegroup="fips-enabled-f9dtl496nd4k" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f",
      "name": "5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f",
      "status": "Failed",
      "startTime": "2026-10-23T13:52:15.708697202Z",
      "endTime": "2026-10-23T14:11:23.696759282Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-f9dtl496nd4k--managed/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-f9dtl496nd4k--managed/providers/Microsoft.Network/networkInterfaces/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to http... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f
            correlation ID: 2e16a443-f7c3-4087-8290-e874ac42ea22
            client request ID: d8c04265-7efa-4df2-aadc-80112544e6fc
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42
Unexpected error:
    <*fmt.wrapError | 0x159fda57b1e0>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-972dcv" in resourcegroup="rg-np-version-upgrade-972dcv-p4hg8tz7gs76" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682",
      "name": "d066a71c-277e-4671-a4dd-16329cfd2682",
      "status": "Failed",
      "startTime": "2026-10-23T14:16:35.253267721Z",
      "endTime": "2026-10-23T14:35:40.055586863Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682
            correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b
            client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42
Unexpected error:
    <*fmt.wrapError | 0x159fda57b1e0>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-972dcv" in resourcegroup="rg-np-version-upgrade-972dcv-p4hg8tz7gs76" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682",
      "name": "d066a71c-277e-4671-a4dd-16329cfd2682",
      "status": "Failed",
      "startTime": "2026-10-23T14:16:35.253267721Z",
      "endTime": "2026-10-23T14:35:40.055586863Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682
            correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b
            client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster "cluster-candidate-4-23-l9g7mz"
Unexpected error:
    <*errors.joinError | 0x2d180ff06060>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg": nodes "y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0x2d180f0a8800>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0x2d180f0a87a0>{
                    msg: "failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0x2d180f0a8780>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0x2d180ff5f680>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster "cluster-candidate-4-23-l9g7mz"
Unexpected error:
    <*errors.joinError | 0x2d180ff06060>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg": nodes "y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0x2d180f0a8800>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0x2d180f0a87a0>{
                    msg: "failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0x2d180f0a8780>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0x2d180ff5f680>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods "policy-test-disallowed" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount "default" not found
Expected
    <string>: pods "policy-test-disallowed" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount "default" not found
to contain substring
    <string>: not from an allowed registry
fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods "policy-test-disallowed" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount "default" not found
Expected
    <string>: pods "policy-test-disallowed" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount "default" not found
to contain substring
    <string>: not from an allowed registry

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool "np-1" for v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x99f918edcc0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20260901" in resourcegroup="v20260901preview-vknb2mpvr5xz" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd",
      "name": "76509bdd-86ae-4734-aa51-d853651ea0fd",
      "status": "Failed",
      "startTime": "2026-10-23T12:59:15.455605547Z",
      "endTime": "2026-10-23T13:18:15.598008978Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd
            correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71
            client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool "np-1" for v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x99f918edcc0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20260901" in resourcegroup="v20260901preview-vknb2mpvr5xz" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd",
      "name": "76509bdd-86ae-4734-aa51-d853651ea0fd",
      "status": "Failed",
      "startTime": "2026-10-23T12:59:15.455605547Z",
      "endTime": "2026-10-23T13:18:15.598008978Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd
            correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71
            client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool "np-d4pgmj" for cluster "cluster-candidate-4-22-d4pgmj"
Unexpected error:
    <*fmt.wrapError | 0x3cda231b9b40>: 
    failed to create NodePool np-d4pgmj: failed waiting for nodepool="np-d4pgmj" for cluster "cluster-candidate-4-22-d4pgmj" in resourcegroup="rg-candidate-4-22-jrp5tdgcl9kv" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0",
      "name": "f456ad88-f3c9-4971-9d6b-4fae5a05e8b0",
      "status": "Failed",
      "startTime": "2026-10-23T12:58:53.825112936Z",
      "endTime": "2026-10-23T13:17:55.550011725Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0
            correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89
            client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool "np-d4pgmj" for cluster "cluster-candidate-4-22-d4pgmj"
Unexpected error:
    <*fmt.wrapError | 0x3cda231b9b40>: 
    failed to create NodePool np-d4pgmj: failed waiting for nodepool="np-d4pgmj" for cluster "cluster-candidate-4-22-d4pgmj" in resourcegroup="rg-candidate-4-22-jrp5tdgcl9kv" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0",
      "name": "f456ad88-f3c9-4971-9d6b-4fae5a05e8b0",
      "status": "Failed",
      "startTime": "2026-10-23T12:58:53.825112936Z",
      "endTime": "2026-10-23T13:17:55.550011725Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0
            correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89
            client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33
occurred

time=2026-10-23T12:34:43.631Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description="Step deploy\n Kind: Helm\n"
time=2026-10-23T12:34:44.030Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-23T12:34:44.030Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-23T12:34:44.168Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller
time=2026-10-23T12:34:44.296Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false
time=2026-10-23T12:34:44.296Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-23T12:34:44.405Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-23T12:39:47.576Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-23T12:39:47.576Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool "np-qptx5c" for cluster "cluster-candidate-5-1-qptx5c"
Unexpected error:
    <*fmt.wrapError | 0x1909134f0c20>: 
    failed to create NodePool np-qptx5c: failed waiting for nodepool="np-qptx5c" for cluster "cluster-candidate-5-1-qptx5c" in resourcegroup="rg-candidate-5-1-6mzh9r6brjfl" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014",
      "name": "1b3eb82d-60f1-4e1d-93f9-e0f48de0d014",
      "status": "Failed",
      "startTime": "2026-10-23T13:34:04.539419115Z",
      "endTime": "2026-10-23T13:53:12.193747284Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014
            correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721
            client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool "np-qptx5c" for cluster "cluster-candidate-5-1-qptx5c"
Unexpected error:
    <*fmt.wrapError | 0x1909134f0c20>: 
    failed to create NodePool np-qptx5c: failed waiting for nodepool="np-qptx5c" for cluster "cluster-candidate-5-1-qptx5c" in resourcegroup="rg-candidate-5-1-6mzh9r6brjfl" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014",
      "name": "1b3eb82d-60f1-4e1d-93f9-e0f48de0d014",
      "status": "Failed",
      "startTime": "2026-10-23T13:34:04.539419115Z",
      "endTime": "2026-10-23T13:53:12.193747284Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014
            correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721
            client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1df6789587c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*verifiers.pollObservationError | 0x1df6789587a0>{
            observation: <*errors.errorString | 0x1df677e0e5e0>{
                s: "dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
            },
            interruption: <*fmt.wrapError | 0x1df678958780>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x1df6776d4500>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x1df678958760>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x1df679343380>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1df6789587c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*verifiers.pollObservationError | 0x1df6789587a0>{
            observation: <*errors.errorString | 0x1df677e0e5e0>{
                s: "dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
            },
            interruption: <*fmt.wrapError | 0x1df678958780>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x1df6776d4500>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x1df678958760>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x1df679343380>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:35:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0830592-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.252:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-558mc", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="a45ad8a2-0299-48d6-b5cc-feba766709c3"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-558mc (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:38:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0830592-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.214:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-9dtps", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="a3569b9f-561d-4f48-9ca7-6bcdc31d1728"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-9dtps (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:38:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0830592-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.214:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-9dtps", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="a3569b9f-561d-4f48-9ca7-6bcdc31d1728"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-9dtps (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:40:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0830592-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.252:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-558mc", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="a45ad8a2-0299-48d6-b5cc-feba766709c3"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-558mc (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentRolloutStuck", cluster="ci01-j0830592-mgmt-2", component="kubernetes-infrastructure", condition="Progressing", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.252:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="progressdeadlineexceeded", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", status="false"
  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentRolloutStuck", cluster="ci01-j0830592-mgmt-2", component="kubernetes-infrastructure", condition="Progressing", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.252:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="progressdeadlineexceeded", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", status="false"
  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:46:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0830592-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.252:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-23T12:46:11Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0830592-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.252:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592", microsoft.metrics_include_label="service", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-23-lrb762"
Expected success, but got an error:
    <*errors.errorString | 0x2b9ddee00df0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-23-lrb762"
Expected success, but got an error:
    <*errors.errorString | 0x2b9ddee00df0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-20-qkcvbq"
Expected success, but got an error:
    <*errors.errorString | 0x49ccf31c560>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-20-qkcvbq"
Expected success, but got an error:
    <*errors.errorString | 0x49ccf31c560>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-5-1-4pwngq"
Expected success, but got an error:
    <*errors.errorString | 0x293284ca6570>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-5-1-4pwngq"
Expected success, but got an error:
    <*errors.errorString | 0x293284ca6570>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

7f52 bbf59cb1  ...nR.......R...
00007ffd4f7fae70: 00007f52 6e7fc640  00007ffd 4f7fb140  @..nR...@..O....
00007ffd4f7fae80: 00007ffd 4f7fafae  00007ffd 4f7fafaf  ...O.......O....
00007ffd4f7fae90: 00007f52 6e7fc640  00007f52 bbed47f5  @..nR....G..R...
00007ffd4f7faea0: 00007f52 bc045c80  00000000 00000120  .\..R... .......
00007ffd4f7faeb0: 00000000 003d0f00  b406abf0 07e46d00  ..=......m......
00007ffd4f7faec0: 00007f52 bbe48740  00000000 00000006  @...R...........
00007ffd4f7faed0: 00000000 1a0a9f00  00007ffd 4f7fb200  ...........O....
00007ffd4f7faee0: 00001a1d 095421e0  00007f52 bbe89646  .!T.....F...R...
00007ffd4f7faef0: 00007f52 bc046e90  00007f52 bbe737f3  .n..R....7..R...
00007ffd4f7faf00: 00000000 00000020  b406abf0 07e46d00   ........m......
00007ffd4f7faf10: 00007f52 6dffc000  00007f52 bbf48ac4  ...mR.......R...
00007ffd4f7faf20: 00000000 00000000  00000000 0000000d  ................

goroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:
runtime.systemstack_switch()
	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8
runtime.newm1(0x0?)
	/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625
runtime.newm(0x5635301?, 0x0, 0xd19a3c0?)
	/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525
runtime.startTemplateThread()
	/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725
runtime.main()
	/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501

goroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.forcegchelper()
	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501
created by runtime.init.7 in goroutine 1
	/usr/lib/golang/src/runtime/proc.go:363 +0x1a

goroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.bgsweep(0x1a1d09692000)
	/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694
runtime.gcenable.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501
created by runtime.gcenable in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:214 +0x66

goroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:
runtime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.(*scavengerState).park(0x17525960)
	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029
runtime.bgscavenge(0x1a1d09692000)
	/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc
runtime.gcenable.gowrap2()
	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501
created by runtime.gcenable in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5

goroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.updateMaxProcsGoroutine()
	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501
created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
	/usr/lib/golang/src/runtime/proc.go:7083 +0x37

rax    0x0
rbx    0x7f52bbe48740
rcx    0x7f52bbed694c
rdx    0x6
rdi    0xa70b
rsi    0xa70b
rbp    0xa70b
rsp    0x7ffd4f7fae30
r8     0x7ffd4f7faf00
r9     0x7f52bbffc4e0
r10    0x8
r11    0x246
r12    0x6
r13    0x7ffd4f7fb200
r14    0x1a1d095421e0
r15    0x10
rip    0x7f52bbed694c
rflags 0x246
cs     0x33
fs     0x0
gs     0x0
Command Error: exit status 2
Deserialization Error: no output from command
7f52 bbf59cb1  ...nR.......R...
00007ffd4f7fae70: 00007f52 6e7fc640  00007ffd 4f7fb140  @..nR...@..O....
00007ffd4f7fae80: 00007ffd 4f7fafae  00007ffd 4f7fafaf  ...O.......O....
00007ffd4f7fae90: 00007f52 6e7fc640  00007f52 bbed47f5  @..nR....G..R...
00007ffd4f7faea0: 00007f52 bc045c80  00000000 00000120  .\..R... .......
00007ffd4f7faeb0: 00000000 003d0f00  b406abf0 07e46d00  ..=......m......
00007ffd4f7faec0: 00007f52 bbe48740  00000000 00000006  @...R...........
00007ffd4f7faed0: 00000000 1a0a9f00  00007ffd 4f7fb200  ...........O....
00007ffd4f7faee0: 00001a1d 095421e0  00007f52 bbe89646  .!T.....F...R...
00007ffd4f7faef0: 00007f52 bc046e90  00007f52 bbe737f3  .n..R....7..R...
00007ffd4f7faf00: 00000000 00000020  b406abf0 07e46d00   ........m......
00007ffd4f7faf10: 00007f52 6dffc000  00007f52 bbf48ac4  ...mR.......R...
00007ffd4f7faf20: 00000000 00000000  00000000 0000000d  ................

goroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:
runtime.systemstack_switch()
	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8
runtime.newm1(0x0?)
	/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625
runtime.newm(0x5635301?, 0x0, 0xd19a3c0?)
	/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525
runtime.startTemplateThread()
	/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725
runtime.main()
	/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501

goroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.forcegchelper()
	/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501
created by runtime.init.7 in goroutine 1
	/usr/lib/golang/src/runtime/proc.go:363 +0x1a

goroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.bgsweep(0x1a1d09692000)
	/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694
runtime.gcenable.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501
created by runtime.gcenable in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:214 +0x66

goroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:
runtime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.(*scavengerState).park(0x17525960)
	/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029
runtime.bgscavenge(0x1a1d09692000)
	/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc
runtime.gcenable.gowrap2()
	/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501
created by runtime.gcenable in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:215 +0xa5

goroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e
runtime.goparkunlock(...)
	/usr/lib/golang/src/runtime/proc.go:468
runtime.updateMaxProcsGoroutine()
	/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501
created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
	/usr/lib/golang/src/runtime/proc.go:7083 +0x37

rax    0x0
rbx    0x7f52bbe48740
rcx    0x7f52bbed694c
rdx    0x6
rdi    0xa70b
rsi    0xa70b
rbp    0xa70b
rsp    0x7ffd4f7fae30
r8     0x7ffd4f7faf00
r9     0x7f52bbffc4e0
r10    0x8
r11    0x246
r12    0x6
r13    0x7ffd4f7fb200
r14    0x1a1d095421e0
r15    0x10
rip    0x7f52bbed694c
rflags 0x246
cs     0x33
fs     0x0
gs     0x0
Command Error: exit status 2
Deserialization Error: no output from command

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x3a796e6c5e8>: 
    resource group "v20260901preview-7822pl2jslgp": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="v20260901" in resourcegroup="v20260901preview-7822pl2jslgp" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1",
      "name": "aa1c3345-b774-4a31-837e-23361b1b3df1",
      "status": "Failed",
      "startTime": "2026-10-23T12:19:48.083110134Z",
      "endTime": "2026-10-23T12:43:54.717460671Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1
            correlation ID: d70e853f-388e-4661-a9cc-9552b311d878
            client request ID: 94a8bbf5-a064-4508-a581-559607551023
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x3a796e6c5e8>: 
    resource group "v20260901preview-7822pl2jslgp": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="v20260901" in resourcegroup="v20260901preview-7822pl2jslgp" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1",
      "name": "aa1c3345-b774-4a31-837e-23361b1b3df1",
      "status": "Failed",
      "startTime": "2026-10-23T12:19:48.083110134Z",
      "endTime": "2026-10-23T12:43:54.717460671Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1
            correlation ID: d70e853f-388e-4661-a9cc-9552b311d878
            client request ID: 94a8bbf5-a064-4508-a581-559607551023
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x236cf2ed0920>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-46ntd4r5fjxr" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866",
      "name": "993068cc-46a0-4e4b-9083-bf129b4ea866",
      "status": "Failed",
      "startTime": "2026-10-23T12:05:04.420745783Z",
      "endTime": "2026-10-23T12:24:12.746383509Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866
            correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447
            client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x236cf2ed0920>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-46ntd4r5fjxr" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866",
      "name": "993068cc-46a0-4e4b-9083-bf129b4ea866",
      "status": "Failed",
      "startTime": "2026-10-23T12:05:04.420745783Z",
      "endTime": "2026-10-23T12:24:12.746383509Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866
            correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447
            client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a
occurred

src/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:
runtime.systemstack_switch()
	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8
runtime.forEachP(...)
	/usr/lib/golang/src/runtime/proc.go:2112
runtime.gcMarkDone()
	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:
runtime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

rax    0x0
rbx    0x7fc7134fd640
rcx    0x7fc75a9bf94c
rdx    0x6
rdi    0xa70a
rsi    0xa711
rbp    0xa711
rsp    0x7fc7134fc790
r8     0x7fc7134fc860
r9     0x7fc75aae54e0
r10    0x8
r11    0x246
r12    0x6
r13    0x7fc7134fcb60
r14    0xccc45e750e0
r15    0xffffffffffffffff
rip    0x7fc75a9bf94c
rflags 0x246
cs     0x33
fs     0x0
gs     0x0
Command Error: exit status 2
Deserialization Error: no output from command
src/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:
runtime.systemstack_switch()
	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8
runtime.forEachP(...)
	/usr/lib/golang/src/runtime/proc.go:2112
runtime.gcMarkDone()
	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:
runtime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e
runtime.gcBgMarkWorker(0xccc45f463f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

rax    0x0
rbx    0x7fc7134fd640
rcx    0x7fc75a9bf94c
rdx    0x6
rdi    0xa70a
rsi    0xa711
rbp    0xa711
rsp    0x7fc7134fc790
r8     0x7fc7134fc860
r9     0x7fc75aae54e0
r10    0x8
r11    0x246
r12    0x6
r13    0x7fc7134fcb60
r14    0xccc45e750e0
r15    0xffffffffffffffff
rip    0x7fc75a9bf94c
rflags 0x246
cs     0x33
fs     0x0
gs     0x0
Command Error: exit status 2
Deserialization Error: no output from command

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x335f1b87d830>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x335f1b87d830>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:
runtime.systemstack_switch()
	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8
runtime.forEachP(...)
	/usr/lib/golang/src/runtime/proc.go:2112
runtime.gcMarkDone()
	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:
runtime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

rax    0x0
rbx    0x7fd81f5da640
rcx    0x7fd866a9c94c
rdx    0x6
rdi    0xa70c
rsi    0xa715
rbp    0xa715
rsp    0x7fd81f5d9790
r8     0x7fd81f5d9860
r9     0x7fd866bc24e0
r10    0x8
r11    0x246
r12    0x6
r13    0x7fd81f5d9b60
r14    0x1077594510e0
r15    0xffffffffffffffff
rip    0x7fd866a9c94c
rflags 0x246
cs     0x33
fs     0x0
gs     0x0
Command Error: exit status 2
Deserialization Error: no output from command
=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:
runtime.systemstack_switch()
	/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8
runtime.forEachP(...)
	/usr/lib/golang/src/runtime/proc.go:2112
runtime.gcMarkDone()
	/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

goroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:
runtime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e
runtime.gcBgMarkWorker(0x1077595223f0)
	/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb
runtime.gcBgMarkStartWorkers.gowrap1()
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7
runtime.goexit({})
	/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/lib/golang/src/runtime/mgc.go:1695 +0x105

rax    0x0
rbx    0x7fd81f5da640
rcx    0x7fd866a9c94c
rdx    0x6
rdi    0xa70c
rsi    0xa715
rbp    0xa715
rsp    0x7fd81f5d9790
r8     0x7fd81f5d9860
r9     0x7fd866bc24e0
r10    0x8
r11    0x246
r12    0x6
r13    0x7fd81f5d9b60
r14    0x1077594510e0
r15    0xffffffffffffffff
rip    0x7fd866a9c94c
rflags 0x246
cs     0x33
fs     0x0
gs     0x0
Command Error: exit status 2
Deserialization Error: no output from command

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x327f918b1650>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x327f918b1650>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x1319a6308af0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x1319a6308af0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x36fd45eaa060>: 
    resource group "rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="np-version-upgrade-cluster-jdxf2b" in resourcegroup="rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a",
      "name": "9861ff20-e277-4831-93f1-1162afbd108a",
      "status": "Failed",
      "startTime": "2026-10-23T13:01:17.842344425Z",
      "endTime": "2026-10-23T13:25:21.427910502Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a
            correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a
            client request ID: 363124c1-5480-4435-98da-729155a01144
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x36fd45eaa060>: 
    resource group "rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="np-version-upgrade-cluster-jdxf2b" in resourcegroup="rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a",
      "name": "9861ff20-e277-4831-93f1-1162afbd108a",
      "status": "Failed",
      "startTime": "2026-10-23T13:01:17.842344425Z",
      "endTime": "2026-10-23T13:25:21.427910502Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a
            correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a
            client request ID: 363124c1-5480-4435-98da-729155a01144
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x33f93d769890>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x33f93d769890>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-21-6jd898"
Expected success, but got an error:
    <*errors.errorString | 0x1690edf5cbb0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-21-6jd898"
Expected success, but got an error:
    <*errors.errorString | 0x1690edf5cbb0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-5-0-ct2hxc"
Expected success, but got an error:
    <*errors.errorString | 0x1c7daddd6340>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-5-0-ct2hxc"
Expected success, but got an error:
    <*errors.errorString | 0x1c7daddd6340>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-22-f9qq4x"
Expected success, but got an error:
    <*errors.errorString | 0x3292fdd6e850>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:223]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-22-f9qq4x"
Expected success, but got an error:
    <*errors.errorString | 0x3292fdd6e850>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x247eb91d9c60>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x247eb91d9c60>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x2fa2959cc2d0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x2fa2959cc2d0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

alert fired 11 time(s) (3 unknown, 8 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T13:16:41Z
  Ended:       2026-10-23T13:26:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6471040-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s", pod="cluster-network-operator-6857fbf8d7-5cgmz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/cluster-network-operator-6857fbf8d7-5cgmz has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-23T13:16:41Z
  Ended:       2026-10-23T13:26:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6471040-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s", pod="ingress-operator-787469b994-cfnzc", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/ingress-operator-787469b994-cfnzc has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-23T13:16:41Z
  Ended:       2026-10-23T13:26:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6471040-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s", pod="dns-operator-6d4899cf84-7qfzv", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/dns-operator-6d4899cf84-7qfzv has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster "cilium-cl"
Unexpected error:
    <*errors.joinError | 0x3fb2ef80a5e8>: 
    VerifyNodesReady failed: there are not ready nodes: cilium-cl-cilium-np-vlnxf-8twbs (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-23T11:44:16Z)
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cl" in resourcegroup="cni-cilium-mj26w2hgsz8z" to finish creating: GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7",
      "name": "e205c3df-0cc8-46bb-84fb-519cf03956a7",
      "status": "Failed",
      "startTime": "2026-10-23T11:25:56.40229713Z",
      "endTime": "2026-10-23T11:45:02.834813538Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine cilium-cl-cilium-np-vlnxf-8twbs: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cl-cilium-np-vlnxf-8twbs: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-mj26w2hgsz8z--managed/cilium-cl-cilium-np-vlnxf-8twbs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-mj26w2hgsz8z--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-vlnxf-8twbs-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://ak... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7
            correlation ID: 98f92748-9012-4fff-87e0-a43e42d009b7
            client request ID: adb6cc59-3412-4269-9828-a7c6939df4dc
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster "cilium-cl"
Unexpected error:
    <*errors.joinError | 0x3fb2ef80a5e8>: 
    VerifyNodesReady failed: there are not ready nodes: cilium-cl-cilium-np-vlnxf-8twbs (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-23T11:44:16Z)
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cl" in resourcegroup="cni-cilium-mj26w2hgsz8z" to finish creating: GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7",
      "name": "e205c3df-0cc8-46bb-84fb-519cf03956a7",
      "status": "Failed",
      "startTime": "2026-10-23T11:25:56.40229713Z",
      "endTime": "2026-10-23T11:45:02.834813538Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine cilium-cl-cilium-np-vlnxf-8twbs: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cl-cilium-np-vlnxf-8twbs: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-mj26w2hgsz8z--managed/cilium-cl-cilium-np-vlnxf-8twbs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-mj26w2hgsz8z--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-vlnxf-8twbs-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://ak... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7
            correlation ID: 98f92748-9012-4fff-87e0-a43e42d009b7
            client request ID: adb6cc59-3412-4269-9828-a7c6939df4dc
occurred

time=2026-10-23T10:04:28.865Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-23T10:04:30.857Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-23T10:04:30.857Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-23T10:04:32.118Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-23T10:04:32.118Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T10:04:32.118Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j4942976-svc
time=2026-10-23T10:19:36.095Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x3a2ce0502aa0>: 
    failed to create NodePool autoscale-noaz: failed waiting for nodepool="autoscale-noaz" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-2q9b4whtt4bx" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6",
      "name": "ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6",
      "status": "Failed",
      "startTime": "2026-10-23T09:46:54.928693784Z",
      "endTime": "2026-10-23T10:05:56.71052837Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6
            correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739
            client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x3a2ce0502aa0>: 
    failed to create NodePool autoscale-noaz: failed waiting for nodepool="autoscale-noaz" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-2q9b4whtt4bx" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6",
      "name": "ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6",
      "status": "Failed",
      "startTime": "2026-10-23T09:46:54.928693784Z",
      "endTime": "2026-10-23T10:05:56.71052837Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6
            correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739
            client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool "np-1" for private keyvault cluster "private-kv-cluster"
Unexpected error:
    <*fmt.wrapError | 0x174cbf30c80>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-kv-cluster" in resourcegroup="private-keyvault-nwv4f2dxcv4b" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5",
      "name": "7be20667-78b5-4033-a3f1-3a092d6628f5",
      "status": "Failed",
      "startTime": "2026-10-23T09:40:10.449361114Z",
      "endTime": "2026-10-23T09:59:13.321219203Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5
            correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f
            client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool "np-1" for private keyvault cluster "private-kv-cluster"
Unexpected error:
    <*fmt.wrapError | 0x174cbf30c80>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-kv-cluster" in resourcegroup="private-keyvault-nwv4f2dxcv4b" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5",
      "name": "7be20667-78b5-4033-a3f1-3a092d6628f5",
      "status": "Failed",
      "startTime": "2026-10-23T09:40:10.449361114Z",
      "endTime": "2026-10-23T09:59:13.321219203Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5
            correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f
            client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x14521ece600>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*verifiers.pollObservationError | 0x14521ece580>{
            observation: <*errors.errorString | 0x14522933200>{
                s: "dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
            },
            interruption: <*fmt.wrapError | 0x14521ece560>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x1452193c140>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x14521ece540>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x14522d0f260>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x14521ece600>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*verifiers.pollObservationError | 0x14521ece580>{
            observation: <*errors.errorString | 0x14522933200>{
                s: "dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
            },
            interruption: <*fmt.wrapError | 0x14521ece560>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x1452193c140>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x14521ece540>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x14522d0f260>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool "np-1" for private keyvault cluster "private-kv-cluster"
Unexpected error:
    <*fmt.wrapError | 0x37701b1c12c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-kv-cluster" in resourcegroup="private-keyvault-snrv785b6qc8" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0",
      "name": "dc59c574-bb7b-4b97-8a05-f66d8dd763d0",
      "status": "Failed",
      "startTime": "2026-10-23T09:27:43.519923521Z",
      "endTime": "2026-10-23T09:46:52.566133312Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0
            correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0
            client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool "np-1" for private keyvault cluster "private-kv-cluster"
Unexpected error:
    <*fmt.wrapError | 0x37701b1c12c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-kv-cluster" in resourcegroup="private-keyvault-snrv785b6qc8" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0",
      "name": "dc59c574-bb7b-4b97-8a05-f66d8dd763d0",
      "status": "Failed",
      "startTime": "2026-10-23T09:27:43.519923521Z",
      "endTime": "2026-10-23T09:46:52.566133312Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0
            correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0
            client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x1d7f216af3c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-sw22pzbqp5qg" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873",
      "name": "57d3456e-a011-478a-861a-14e4fe67f873",
      "status": "Failed",
      "startTime": "2026-10-23T09:12:07.672503322Z",
      "endTime": "2026-10-23T09:31:10.853698527Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873
            correlation ID: 62c61aed-7617-418e-886e-be6374e7f22a
            client request ID: eb9543dc-eb4b-4b68-a493-4efa1c23921e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x1d7f216af3c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-sw22pzbqp5qg" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873",
      "name": "57d3456e-a011-478a-861a-14e4fe67f873",
      "status": "Failed",
      "startTime": "2026-10-23T09:12:07.672503322Z",
      "endTime": "2026-10-23T09:31:10.853698527Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873
            correlation ID: 62c61aed-7617-418e-886e-be6374e7f22a
            client request ID: eb9543dc-eb4b-4b68-a493-4efa1c23921e
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.004s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x231711c56ee0>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893->172.30.0.10:53: read: connection refused",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.004s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x231711c56ee0>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893->172.30.0.10:53: read: connection refused",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool "np-1" for v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0xfba86cf63e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20260901" in resourcegroup="v20260901preview-gkpx2dhvd6bm" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd",
      "name": "7f0ee636-256c-422d-a7d4-36addd512bcd",
      "status": "Failed",
      "startTime": "2026-10-23T09:14:45.906565692Z",
      "endTime": "2026-10-23T09:33:49.808866171Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd
            correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175
            client request ID: f1495231-4c88-4b44-9a23-53972ede44a2
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool "np-1" for v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0xfba86cf63e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20260901" in resourcegroup="v20260901preview-gkpx2dhvd6bm" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd",
      "name": "7f0ee636-256c-422d-a7d4-36addd512bcd",
      "status": "Failed",
      "startTime": "2026-10-23T09:14:45.906565692Z",
      "endTime": "2026-10-23T09:33:49.808866171Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd
            correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175
            client request ID: f1495231-4c88-4b44-9a23-53972ede44a2
occurred

time=2026-10-23T08:49:29.045Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description="Step deploy\n Kind: Helm\n"
time=2026-10-23T08:49:29.470Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-23T08:49:29.470Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-23T08:49:29.655Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller
time=2026-10-23T08:49:29.801Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false
time=2026-10-23T08:49:29.801Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-23T08:49:29.867Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-23T08:54:32.503Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-23T08:54:32.504Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x3068a4b9eb40>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-z7kvkzn6t5zb" to finish creating: GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896",
      "name": "90af8b99-1ac0-4bf4-84da-3d4413591896",
      "status": "Failed",
      "startTime": "2026-10-23T09:06:48.532373154Z",
      "endTime": "2026-10-23T09:25:48.512267136Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896
            correlation ID: f7aa4bb1-0464-427f-ba40-60c91211299f
            client request ID: 1f07b0d4-9ffc-4cd6-8bff-a623d13c31fb
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x3068a4b9eb40>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-z7kvkzn6t5zb" to finish creating: GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896",
      "name": "90af8b99-1ac0-4bf4-84da-3d4413591896",
      "status": "Failed",
      "startTime": "2026-10-23T09:06:48.532373154Z",
      "endTime": "2026-10-23T09:25:48.512267136Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1950208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90af8b99-1ac0-4bf4-84da-3d4413591896
            correlation ID: f7aa4bb1-0464-427f-ba40-60c91211299f
            client request ID: 1f07b0d4-9ffc-4cd6-8bff-a623d13c31fb

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T08:40:51Z
  Ended:       2026-10-23T08:58:52Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j1650944-svc", component="backend", name="csstatedump", region="canadacentral", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-23T08:40:51Z
  Ended:       2026-10-23T08:57:52Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j1650944-svc", component="backend", name="clusterresources", region="canadacentral", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool "nodepool-128GiB" with 128GiB osDisk
Unexpected error:
    <*fmt.wrapError | 0x3b310ced2400>: 
    failed to create NodePool nodepool-128GiB: failed waiting for nodepool="nodepool-128GiB" for cluster "hcp-cluster-np-128" in resourcegroup="clusternp128-d2zgjcg49b99" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b",
      "name": "8b884d06-af05-4a79-8c59-961aac29861b",
      "status": "Failed",
      "startTime": "2026-10-23T07:12:27.383685402Z",
      "endTime": "2026-10-23T07:31:36.955210198Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b
            correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c
            client request ID: f057ae85-7143-412c-b3d7-6a49973e5447
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool "nodepool-128GiB" with 128GiB osDisk
Unexpected error:
    <*fmt.wrapError | 0x3b310ced2400>: 
    failed to create NodePool nodepool-128GiB: failed waiting for nodepool="nodepool-128GiB" for cluster "hcp-cluster-np-128" in resourcegroup="clusternp128-d2zgjcg49b99" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b",
      "name": "8b884d06-af05-4a79-8c59-961aac29861b",
      "status": "Failed",
      "startTime": "2026-10-23T07:12:27.383685402Z",
      "endTime": "2026-10-23T07:31:36.955210198Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b
            correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c
            client request ID: f057ae85-7143-412c-b3d7-6a49973e5447
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-4m47cf" for cluster "cluster-hs-pre-4m47cf"
Unexpected error:
    <*fmt.wrapError | 0x25f14c702960>: 
    failed to create NodePool np-4m47cf: failed waiting for nodepool="np-4m47cf" for cluster "cluster-hs-pre-4m47cf" in resourcegroup="rg-hs-pre--phztwwdrkw7h" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed",
      "name": "d4f6c18f-891a-4c33-a2ba-8972525d31ed",
      "status": "Failed",
      "startTime": "2026-10-23T07:15:22.43923977Z",
      "endTime": "2026-10-23T07:34:26.438466293Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed
            correlation ID: dcef897a-e7d0-4e15-8934-33f203c26727
            client request ID: f4ae8e10-89f7-445a-9181-9c41d68ea128
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-4m47cf" for cluster "cluster-hs-pre-4m47cf"
Unexpected error:
    <*fmt.wrapError | 0x25f14c702960>: 
    failed to create NodePool np-4m47cf: failed waiting for nodepool="np-4m47cf" for cluster "cluster-hs-pre-4m47cf" in resourcegroup="rg-hs-pre--phztwwdrkw7h" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed",
      "name": "d4f6c18f-891a-4c33-a2ba-8972525d31ed",
      "status": "Failed",
      "startTime": "2026-10-23T07:15:22.43923977Z",
      "endTime": "2026-10-23T07:34:26.438466293Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed
            correlation ID: dcef897a-e7d0-4e15-8934-33f203c26727
            client request ID: f4ae8e10-89f7-445a-9181-9c41d68ea128
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain
Expected success, but got an error:
    <*fmt.wrapError | 0x1f9142952260>: 
    DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x1f9142952240>{
            msg: "DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain
Expected success, but got an error:
    <*fmt.wrapError | 0x1f9142952260>: 
    DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x1f9142952240>{
            msg: "DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x23df1c5ad4a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x23df1caefde0>{
            s: "dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x23df1c5ad4a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x23df1caefde0>{
            s: "dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x325a69290ac0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-vq4tbh9zl5hx" to finish creating: GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7",
      "name": "608a5015-c575-47f4-96b6-93f98f6b51f7",
      "status": "Failed",
      "startTime": "2026-10-23T06:24:56.751408267Z",
      "endTime": "2026-10-23T06:43:58.642073257Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7
            correlation ID: aa734835-f56e-4cf5-867a-994867172e04
            client request ID: 977d7eeb-364d-4bbf-931a-b35202d4aeda
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x325a69290ac0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-vq4tbh9zl5hx" to finish creating: GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7",
      "name": "608a5015-c575-47f4-96b6-93f98f6b51f7",
      "status": "Failed",
      "startTime": "2026-10-23T06:24:56.751408267Z",
      "endTime": "2026-10-23T06:43:58.642073257Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3270400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/608a5015-c575-47f4-96b6-93f98f6b51f7
            correlation ID: aa734835-f56e-4cf5-867a-994867172e04
            client request ID: 977d7eeb-364d-4bbf-931a-b35202d4aeda

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-xhrpwn"
Unexpected error:
    <*fmt.wrapError | 0x1788046d3680>: 
    DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x178804888840>{
            msg: "DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-xhrpwn"
Unexpected error:
    <*fmt.wrapError | 0x1788046d3680>: 
    DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x178804888840>{
            msg: "DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade
Expected
    <set.Set[string] | len:0>: {}
not to be empty
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade
Expected
    <set.Set[string] | len:0>: {}
not to be empty

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x9a09bb1e000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {"discovered_targets":0,"omitted_targets":0,"submitted_targets":0,"partial":true,"reported_targets":0,"outcome":"failed","unavailable":"discovery_failed","canceled":false,"age_seconds":1300,"cadence_seconds":300,"elapsed_ms":3,"roles":{},"http_success":0,"http_failed":0,"tcp_connected":0,"tcp_refused":0,"tcp_timeout":0,"tcp_failed":0,"retired_failed":0,"dns_success":0,"dns_failed":0,"dns_reported":0,"dns_expected":0,"truncated":0,"evidence_errors":0}
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}",
        err: <*errors.errorString | 0x9a09aa7e480>{
            s: "router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x9a09bb1e000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {"discovered_targets":0,"omitted_targets":0,"submitted_targets":0,"partial":true,"reported_targets":0,"outcome":"failed","unavailable":"discovery_failed","canceled":false,"age_seconds":1300,"cadence_seconds":300,"elapsed_ms":3,"roles":{},"http_success":0,"http_failed":0,"tcp_connected":0,"tcp_refused":0,"tcp_timeout":0,"tcp_failed":0,"retired_failed":0,"dns_success":0,"dns_failed":0,"dns_reported":0,"dns_expected":0,"truncated":0,"evidence_errors":0}
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}",
        err: <*errors.errorString | 0x9a09aa7e480>{
            s: "router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade
Expected
    <set.Set[string] | len:0>: {}
not to be empty
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade
Expected
    <set.Set[string] | len:0>: {}
not to be empty

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes
Unexpected error:
    <*fmt.wrapError | 0x3221f83f4600>: 
    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled
    {
        msg: "VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled",
        err: <*errors.errorString | 0x3221fa87e2b0>{
            s: "DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes
Unexpected error:
    <*fmt.wrapError | 0x3221f83f4600>: 
    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled
    {
        msg: "VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled",
        err: <*errors.errorString | 0x3221fa87e2b0>{
            s: "DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.
node labels/taints should appear on the intended node pools of cluster "agg-cluster"
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:
node pool "agg-np-a" should have at least 2 nodes (autoscaling min), found 0
Expected
    <int>: 0
to be >=
    <int>: 2
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.
node labels/taints should appear on the intended node pools of cluster "agg-cluster"
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:
node pool "agg-np-a" should have at least 2 nodes (autoscaling min), found 0
Expected
    <int>: 0
to be >=
    <int>: 2

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T06:49:50Z
  Ended:       2026-10-23T06:56:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j3270400-mgmt-2", component="kubernetes-infrastructure", container="manager", endpoint="http", environment="ci01", instance="10.128.64.213:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3270400/providers/microsoft.monitor/accounts/hcps-j3270400", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le", pod="capi-provider-7d64b7b878-2mwb4", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="40ed8520-738a-4471-a9a5-0fd0a1d23770"
  Description: Pod ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le/capi-provider-7d64b7b878-2mwb4 (manager) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-23T06:49:50Z
  Ended:       2026-10-23T06:56:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j3270400-mgmt-2", component="kubernetes-infrastructure", container="manager", endpoint="http", environment="ci01", instance="10.128.64.213:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3270400/providers/microsoft.monitor/accounts/hcps-j3270400", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le", pod="capi-provider-7d64b7b878-2mwb4", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="40ed8520-738a-4471-a9a5-0fd0a1d23770"
  Description: Pod ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le/capi-provider-7d64b7b878-2mwb4 (manager) is in waiting state (reason: "CrashLoopBackOff").

time=2026-10-23T04:51:07.246Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description="Step mirror-velero-image\n  Kind: ImageMirror\n  From {<nil> velero.server.registry <nil>}:{<nil> velero.server.repository <nil>}@{<nil> velero.server.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-10-23T04:52:17.695Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\nFetch pull secret for source registry registry.stage.redhat.io from arohcpdev-global KV.\nLogging into target ACR arohcpsvcdev.\nLogin Succeeded\nMirroring image registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc.\nThe image will still be available under it's original digest sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc in the target registry.\nError response from registry: recognizable error message not found: GET \"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull&service=docker-registry\": response status code 503: Service Unavailable\nCommand failed (attempt 1/5). Retrying in 2s...\nError response from registry: recognizable error message not found: GET \"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull&service=docker-registry\": response status code 503: Service Unavailable\nCommand failed (attempt 2/5). Retrying in 4s...\nError response from registry: recognizable error message not found: GET \"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull&service=docker-registry\": response status code 503: Service Unavailable\nCommand failed (attempt 3/5). Retrying in 8s...\nError response from registry: recognizable error message not found: GET \"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull&service=docker-registry\": response status code 503: Service Unavailable\nCommand failed (attempt 4/5). Retrying in 16s...\nError response from registry: recognizable error message not found: GET \"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull&service=docker-registry\": response status code 503: Service Unavailable\nCommand failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc --from-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json --to-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json\n exit status 1"

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x12108ffe4840>: 
    failed to create NodePool autoscale-az: failed waiting for nodepool="autoscale-az" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-gmjhx5ggwwt9" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904",
      "name": "74fee9f7-b402-4d5d-b683-6e4318e06904",
      "status": "Failed",
      "startTime": "2026-10-23T03:34:18.354098626Z",
      "endTime": "2026-10-23T03:53:18.691661116Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904
            correlation ID: b1f52151-27ba-447c-87a0-0859e9b312e7
            client request ID: f918cf71-16c2-4813-9b6f-6f13fae950d6
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x12108ffe4840>: 
    failed to create NodePool autoscale-az: failed waiting for nodepool="autoscale-az" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-gmjhx5ggwwt9" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904",
      "name": "74fee9f7-b402-4d5d-b683-6e4318e06904",
      "status": "Failed",
      "startTime": "2026-10-23T03:34:18.354098626Z",
      "endTime": "2026-10-23T03:53:18.691661116Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904
            correlation ID: b1f52151-27ba-447c-87a0-0859e9b312e7
            client request ID: f918cf71-16c2-4813-9b6f-6f13fae950d6
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0x2987b030ca40>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-rzw7qwnn5qjc" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed",
      "name": "76e4548a-7fa4-49e0-977c-056006f3a3ed",
      "status": "Failed",
      "startTime": "2026-10-23T03:22:03.46163902Z",
      "endTime": "2026-10-23T03:51:08.692840996Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed
            correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf
            client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0x2987b030ca40>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-rzw7qwnn5qjc" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed",
      "name": "76e4548a-7fa4-49e0-977c-056006f3a3ed",
      "status": "Failed",
      "startTime": "2026-10-23T03:22:03.46163902Z",
      "endTime": "2026-10-23T03:51:08.692840996Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed
            correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf
            client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T03:34:48Z
  Ended:       2026-10-23T03:58:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2625664-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tk89mvlqfc5cf20qag1guihinhj7c0v-z6o9i3i9h0a8n5p", pod="router-5cfd955d57-tpd7s", region="canadacentral", severity="warning"
  Description: Pod ocm-arohcpci01-2tk89mvlqfc5cf20qag1guihinhj7c0v-z6o9i3i9h0a8n5p/router-5cfd955d57-tpd7s has been in a non-ready state for longer than 5 minutes.

time=2026-10-23T02:42:44.129Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description="Step deploy\n Kind: Helm\n"
time=2026-10-23T02:42:44.420Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-23T02:42:44.420Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T02:42:44.639Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller
time=2026-10-23T02:42:44.852Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false
time=2026-10-23T02:42:44.852Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T02:42:44.905Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T02:47:48.850Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-23T02:47:48.850Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-23T02:40:08Z
  Ended:       2026-10-23T02:49:10Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j3977984-mgmt-2", component="monitoring-infrastructure", region="canadacentral", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.

time=2026-10-23T01:37:22.600Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-23T01:37:23.985Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-23T01:37:23.985Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-23T01:37:24.555Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-23T01:37:24.555Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-23T01:37:24.555Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j4358528-svc
time=2026-10-23T01:52:27.816Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"

time=2026-10-23T01:14:47.048Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description="Step deploy\n Kind: Helm\n"
time=2026-10-23T01:14:47.425Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-23T01:14:47.425Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T01:14:47.606Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller
time=2026-10-23T01:14:47.801Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false
time=2026-10-23T01:14:47.801Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T01:14:47.849Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1
time=2026-10-23T01:19:51.482Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-23T01:19:51.483Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool "np-1" for autoscaling cluster
Unexpected error:
    <*fmt.wrapError | 0x24b9ff37f340>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-xqmhq7fznl4t" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb",
      "name": "90f18bf3-8ce0-485d-ade8-a5cce140bcbb",
      "status": "Failed",
      "startTime": "2026-10-23T00:52:55.985747163Z",
      "endTime": "2026-10-23T01:11:56.482302149Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb
            correlation ID: 980d0943-0968-489f-acea-10a235919d44
            client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool "np-1" for autoscaling cluster
Unexpected error:
    <*fmt.wrapError | 0x24b9ff37f340>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-xqmhq7fznl4t" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb",
      "name": "90f18bf3-8ce0-485d-ade8-a5cce140bcbb",
      "status": "Failed",
      "startTime": "2026-10-23T00:52:55.985747163Z",
      "endTime": "2026-10-23T01:11:56.482302149Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb
            correlation ID: 980d0943-0968-489f-acea-10a235919d44
            client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool "arm64-vm-np-1" with VM size "Standard_D2plds_v6"
Unexpected error:
    <*fmt.wrapError | 0x5006b07f360>: 
    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool="arm64-vm-np-1" for cluster "arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-lzb9ghdnlbff" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4",
      "name": "26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4",
      "status": "Failed",
      "startTime": "2026-10-23T00:53:35.812103666Z",
      "endTime": "2026-10-23T01:12:35.662761223Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4
            correlation ID: c531ae4a-649c-4e3c-a628-8707dc54dc8b
            client request ID: e95dcc96-488a-4fef-96e9-1e87cdadcd34
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool "arm64-vm-np-1" with VM size "Standard_D2plds_v6"
Unexpected error:
    <*fmt.wrapError | 0x5006b07f360>: 
    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool="arm64-vm-np-1" for cluster "arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-lzb9ghdnlbff" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4",
      "name": "26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4",
      "status": "Failed",
      "startTime": "2026-10-23T00:53:35.812103666Z",
      "endTime": "2026-10-23T01:12:35.662761223Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26e0fd53-6a40-46c1-ae9b-bb76ceedf3e4
            correlation ID: c531ae4a-649c-4e3c-a628-8707dc54dc8b
            client request ID: e95dcc96-488a-4fef-96e9-1e87cdadcd34
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0xd5c47f9e8e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-4ffqgf6wvkdd" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005",
      "name": "de16b959-58ce-4c0d-a6d8-56ec91482005",
      "status": "Failed",
      "startTime": "2026-10-23T00:56:40.180542426Z",
      "endTime": "2026-10-23T01:15:42.00975543Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005
            correlation ID: 5bb64d81-21d2-4966-9cfc-0f0641febce3
            client request ID: 70b4240c-c009-4928-97c5-bfd8b74a6560
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0xd5c47f9e8e0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-4ffqgf6wvkdd" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005",
      "name": "de16b959-58ce-4c0d-a6d8-56ec91482005",
      "status": "Failed",
      "startTime": "2026-10-23T00:56:40.180542426Z",
      "endTime": "2026-10-23T01:15:42.00975543Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de16b959-58ce-4c0d-a6d8-56ec91482005
            correlation ID: 5bb64d81-21d2-4966-9cfc-0f0641febce3
            client request ID: 70b4240c-c009-4928-97c5-bfd8b74a6560
occurred