time=2026-10-16T23:16:01.424Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T23:16:02.804Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found"

time=2026-10-16T23:22:36.468Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description="Step storageclass\n Kind: Helm\n"
time=2026-10-16T23:22:36.986Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values="map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]"
time=2026-10-16T23:22:36.986Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-16T23:22:37.251Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc
time=2026-10-16T23:22:37.674Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false
time=2026-10-16T23:22:37.674Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-16T23:22:37.803Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1
time=2026-10-16T23:23:44.090Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err="failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"
time=2026-10-16T23:23:44.090Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"

time=2026-10-16T23:13:05.250Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-16T23:13:06.422Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-16T23:13:06.422Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-16T23:13:07.473Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-16T23:13:07.473Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-16T23:13:07.473Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j6132992-svc
time=2026-10-16T23:28:10.126Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"

time=2026-10-16T22:41:06.800Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T22:41:08.120Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI
Unexpected error:
    <*errors.joinError | 0x165441e2cf48>: 
    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]
    {
        errs: [
            <*fmt.wrapError | 0x165441dab3a0>{
                msg: "VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]",
                err: <*fmt.wrapError | 0x165441dab1c0>{
                    msg: "connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]",
                    err: <*errors.errorString | 0x1654427290b0>{
                        s: "test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]",
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI
Unexpected error:
    <*errors.joinError | 0x165441e2cf48>: 
    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]
    {
        errs: [
            <*fmt.wrapError | 0x165441dab3a0>{
                msg: "VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]",
                err: <*fmt.wrapError | 0x165441dab1c0>{
                    msg: "connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]",
                    err: <*errors.errorString | 0x1654427290b0>{
                        s: "test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]",
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x33c730fe0780>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-2q8nt57gw7c4" to finish creating: GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa",
      "name": "f4f78025-9771-4e07-9a76-cc316ad5bcaa",
      "status": "Failed",
      "startTime": "2026-10-16T23:21:44.60401103Z",
      "endTime": "2026-10-16T23:40:48.245826795Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-2q8nt57gw7c4--managed/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Network/networkInterfaces/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa
            correlation ID: 57a29177-39d2-4d3f-ac56-44b9805e57f9
            client request ID: 8065bb9a-6902-4747-a2b4-2914d4dc8e70
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0x33c730fe0780>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-2q8nt57gw7c4" to finish creating: GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa",
      "name": "f4f78025-9771-4e07-9a76-cc316ad5bcaa",
      "status": "Failed",
      "startTime": "2026-10-16T23:21:44.60401103Z",
      "endTime": "2026-10-16T23:40:48.245826795Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-2q8nt57gw7c4--managed/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Network/networkInterfaces/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa
            correlation ID: 57a29177-39d2-4d3f-ac56-44b9805e57f9
            client request ID: 8065bb9a-6902-4747-a2b4-2914d4dc8e70
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1
Unexpected error:
    <*fmt.wrapError | 0x2fa3cd2a3c20>: 
    failed to create NodePool ea-np-1: failed waiting for nodepool="ea-np-1" for cluster "ea-cluster" in resourcegroup="external-auth-cluster-wl8xt5tlkcq7" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5",
      "name": "63825d69-d610-4421-b3ee-50ac654908c5",
      "status": "Failed",
      "startTime": "2026-10-16T21:00:14.715888281Z",
      "endTime": "2026-10-16T21:19:17.454369198Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ea-cluster-ea-np-1-rnlcf-ttbgr: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-wl8xt5tlkcq7--managed/ea-cluster-ea-np-1-rnlcf-ttbgr-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wl8xt5tlkcq7--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-rnlcf-ttbgr-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5
            correlation ID: 2667b7c1-2456-4fb3-a1b3-9f090199ed55
            client request ID: 1483f922-ab7c-4902-933a-1a3aef1cf957
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1
Unexpected error:
    <*fmt.wrapError | 0x2fa3cd2a3c20>: 
    failed to create NodePool ea-np-1: failed waiting for nodepool="ea-np-1" for cluster "ea-cluster" in resourcegroup="external-auth-cluster-wl8xt5tlkcq7" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5",
      "name": "63825d69-d610-4421-b3ee-50ac654908c5",
      "status": "Failed",
      "startTime": "2026-10-16T21:00:14.715888281Z",
      "endTime": "2026-10-16T21:19:17.454369198Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ea-cluster-ea-np-1-rnlcf-ttbgr: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-wl8xt5tlkcq7--managed/ea-cluster-ea-np-1-rnlcf-ttbgr-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wl8xt5tlkcq7--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-rnlcf-ttbgr-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5
            correlation ID: 2667b7c1-2456-4fb3-a1b3-9f090199ed55
            client request ID: 1483f922-ab7c-4902-933a-1a3aef1cf957
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x1d04d19fc660>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-kjxkhz" in resourcegroup="rg-np-version-upgrade-kjxkhz-ldkgrb6cvc64" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae",
      "name": "eec0983a-6d6f-417c-a350-df06c778abae",
      "status": "Failed",
      "startTime": "2026-10-16T21:16:56.042187584Z",
      "endTime": "2026-10-16T21:35:57.540237831Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae
            correlation ID: 62969ebc-fdd1-4a1a-808f-bd02fcd158bd
            client request ID: f0c87d09-298f-426d-885a-00fac76d2bf1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x1d04d19fc660>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-kjxkhz" in resourcegroup="rg-np-version-upgrade-kjxkhz-ldkgrb6cvc64" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae",
      "name": "eec0983a-6d6f-417c-a350-df06c778abae",
      "status": "Failed",
      "startTime": "2026-10-16T21:16:56.042187584Z",
      "endTime": "2026-10-16T21:35:57.540237831Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae
            correlation ID: 62969ebc-fdd1-4a1a-808f-bd02fcd158bd
            client request ID: f0c87d09-298f-426d-885a-00fac76d2bf1
occurred

alert fired 3 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T21:21:48Z
  Ended:       2026-10-16T21:29:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4534656-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q", pod="ignition-server-proxy-d589845df-77l2s", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q/ignition-server-proxy-d589845df-77l2s has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T21:21:48Z
  Ended:       2026-10-16T21:29:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4534656-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q", pod="ignition-server-proxy-d589845df-fjmtf", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q/ignition-server-proxy-d589845df-fjmtf has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-16T21:21:48Z
  Ended:       2026-10-16T21:34:57Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4534656-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q", pod="network-node-identity-6bdbcc6c4c-jtfx4", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q/network-node-identity-6bdbcc6c4c-jtfx4 has been in a non-ready state for longer than 5 minutes.

time=2026-10-16T17:28:31.305Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T17:28:32.589Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found"

time=2026-10-16T17:08:49.998Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T17:08:51.188Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x3a1e5c39e140>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-lsp7k9q4q2pz" to finish creating: GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f",
      "name": "eeecd748-7cd9-428c-bdad-6172bba0e52f",
      "status": "Failed",
      "startTime": "2026-10-16T17:39:40.172421101Z",
      "endTime": "2026-10-16T17:58:48.49629868Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-pb4xk-8sfrz to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-lsp7k9q4q2pz--managed/ingress-cert-le-np-1-pb4xk-8sfrz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-pb4xk-8sfrz-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f
            correlation ID: 3ee22f63-ca3a-4588-b9c5-ce97876471df
            client request ID: a644f8be-e878-4c8d-99e9-321877a22983
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x3a1e5c39e140>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-lsp7k9q4q2pz" to finish creating: GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f",
      "name": "eeecd748-7cd9-428c-bdad-6172bba0e52f",
      "status": "Failed",
      "startTime": "2026-10-16T17:39:40.172421101Z",
      "endTime": "2026-10-16T17:58:48.49629868Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-pb4xk-8sfrz to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-lsp7k9q4q2pz--managed/ingress-cert-le-np-1-pb4xk-8sfrz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-pb4xk-8sfrz-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f
            correlation ID: 3ee22f63-ca3a-4588-b9c5-ce97876471df
            client request ID: a644f8be-e878-4c8d-99e9-321877a22983

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T17:47:40Z
  Ended:       2026-10-16T17:58:40Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j1762432-svc", component="backend", name="clusterresources", region="canadacentral", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T17:57:51Z
  Ended:       2026-10-16T18:04:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1762432-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tg1320l1nm4kt4ln02jlh4ms9qcvrv7-v20260901", pod="openshift-apiserver-7f89dd8c4b-sck4g", region="canadacentral", severity="warning"
  Description: Pod ocm-arohcpci01-2tg1320l1nm4kt4ln02jlh4ms9qcvrv7-v20260901/openshift-apiserver-7f89dd8c4b-sck4g has been in a non-ready state for longer than 5 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T17:00:31Z
  Ended:       2026-10-16T17:16:34Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j5648768-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T17:00:31Z
  Ended:       2026-10-16T17:14:30Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j5648768-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3222464a4800>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z",
        err: <*errors.errorString | 0x322245a3b8b0>{
            s: "dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3222464a4800>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z",
        err: <*errors.errorString | 0x322245a3b8b0>{
            s: "dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z",
        },
    }
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T17:29:20Z
  Ended:       2026-10-16T17:35:23Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1109632-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tg0fmg5i9694ug5ff0i7ru6ntjv7r87-v2a4k0f7j4n8v4m", pod="etcd-0", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tg0fmg5i9694ug5ff0i7ru6ntjv7r87-v2a4k0f7j4n8v4m/etcd-0 has been in a non-ready state for longer than 5 minutes.

time=2026-10-16T15:46:44.545Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description="Step storageclass\n Kind: Helm\n"
time=2026-10-16T15:46:45.068Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values="map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]"
time=2026-10-16T15:46:45.068Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2
time=2026-10-16T15:46:45.193Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc
time=2026-10-16T15:46:45.343Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false
time=2026-10-16T15:46:45.343Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2
time=2026-10-16T15:46:45.452Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2
time=2026-10-16T15:47:32.298Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err="failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"
time=2026-10-16T15:47:32.298Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x225983c28ba0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T15:59:20.1954973Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x225983c28b60>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x225983a80280>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x225983c28b20>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x22598544d170>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x225983c28ba0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T15:59:20.1954973Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x225983c28b60>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x225983a80280>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x225983c28b20>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x22598544d170>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

time=2026-10-16T16:06:45.672Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description="Step deploy\n Kind: Helm\n"
time=2026-10-16T16:06:46.344Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values="map[azureArmHelperIdentityCertName:armHelperPoolCert-38 azureArmHelperIdentityClientId:6608bb27-dfa1-4c02-8395-4ca010bd73db azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:d0cc3060-1e84-4147-a32a-544fe7e53387 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-16 azureMiMockServicePrincipalClientId:33e65307-a696-491e-831f-42341ce3a9b5 azureMiMockServicePrincipalPrincipalId:b26c2343-0863-4a55-bb7e-dfb2544999c1 azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j7660416 cosmosDatabaseUrl:https://arohcpci01-rp-j7660416.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:e9d0c188179de6a6eb745bf8b91e2a9cf54424a74b55ce29ef87ab06cf658ddb] imageDigest:sha256:e9d0c188179de6a6eb745bf8b91e2a9cf54424a74b55ce29ef87ab06cf658ddb imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj7660416.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj7660416.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:0531f05d-8afc-4435-86c7-b9a82d26a229 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:westus3 replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]"
time=2026-10-16T16:06:46.344Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy
time=2026-10-16T16:06:46.586Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service
time=2026-10-16T16:06:46.981Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false
time=2026-10-16T16:06:46.981Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy
time=2026-10-16T16:06:47.117Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy
time=2026-10-16T16:16:56.303Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err="resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\ncontext deadline exceeded"
time=2026-10-16T16:16:56.304Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err="error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\ncontext deadline exceeded"

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:17:41Z
  Severity:    Sev2
  Labels:      alertname="MaestroServerNoReadyReplicas", cluster="ci01-j7660416-svc", component="maestro", container="kube-state-metrics", deployment="maestro", endpoint="http", environment="ci01", instance="10.128.64.75:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7660416/providers/microsoft.monitor/accounts/services-j7660416", namespace="maestro", pod="arohcp-monitor-kube-state-metrics-79b88f5699-llr7w", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="critical"
  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:17:41Z
  Severity:    Sev2
  Labels:      alertname="MaestroServerNoReadyReplicas", cluster="ci01-j7660416-svc", component="maestro", container="kube-state-metrics", deployment="maestro", endpoint="http", environment="ci01", instance="10.128.64.75:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7660416/providers/microsoft.monitor/accounts/services-j7660416", namespace="maestro", pod="arohcp-monitor-kube-state-metrics-79b88f5699-llr7w", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="critical"
  Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.

alert fired 18 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:03:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-mgmt-1", component="kubernetes-infrastructure", namespace="open-cluster-management-agent-addon", pod="klusterlet-addon-workmgr-c4d6b786-548pt", region="westus3", severity="warning"
  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-c4d6b786-548pt has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="aks-secrets-store-csi-driver-5lt2p", region="westus3", severity="warning"
  Description: Pod kube-system/aks-secrets-store-csi-driver-5lt2p has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="aks-secrets-store-csi-driver-nw2d2", region="westus3", severity="warning"
  Description: Pod kube-system/aks-secrets-store-csi-driver-nw2d2 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="aks-secrets-store-csi-driver-rsnz7", region="westus3", severity="warning"
  Description: Pod kube-system/aks-secrets-store-csi-driver-rsnz7 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="ama-metrics-node-q6lgt", region="westus3", severity="warning"
  Description: Pod kube-system/ama-metrics-node-q6lgt has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="eraser-aks-system-27216613-vmss000000-jg5jx", region="westus3", severity="warning"
  Description: Pod kube-system/eraser-aks-system-27216613-vmss000000-jg5jx has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="hubble-relay-58594b5dd7-c85j5", region="westus3", severity="warning"
  Description: Pod kube-system/hubble-relay-58594b5dd7-c85j5 has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="ama-metrics-node-9j44l", region="westus3", severity="warning"
  Description: Pod kube-system/ama-metrics-node-9j44l has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="eraser-aks-system-27216613-vmss000002-wgh5j", region="westus3", severity="warning"
  Description: Pod kube-system/eraser-aks-system-27216613-vmss000002-wgh5j has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="eraser-aks-system-27216613-vmss000001-zvc5c", region="westus3", severity="warning"
  Description: Pod kube-system/eraser-aks-system-27216613-vmss000001-zvc5c has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:05:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="ama-metrics-node-bjvb8", region="westus3", severity="warning"
  Description: Pod kube-system/ama-metrics-node-bjvb8 has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:13:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="clusters-service-6cb67d7f44-f4tbd", region="westus3", severity="warning"
  Description: Pod clusters-service/clusters-service-6cb67d7f44-f4tbd has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:13:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="sessiongate", pod="sessiongate-77fbb4f5f5-2fjpq", region="westus3", severity="warning"
  Description: Pod sessiongate/sessiongate-77fbb4f5f5-2fjpq has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:13:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="clusters-service-6cb67d7f44-89fpd", region="westus3", severity="warning"
  Description: Pod clusters-service/clusters-service-6cb67d7f44-89fpd has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:13:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="clusters-service-6cb67d7f44-g6qp6", region="westus3", severity="warning"
  Description: Pod clusters-service/clusters-service-6cb67d7f44-g6qp6 has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:13:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="maestro", pod="maestro-7dfb5b7d6b-dgw7q", region="westus3", severity="warning"
  Description: Pod maestro/maestro-7dfb5b7d6b-dgw7q has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:13:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="maestro", pod="maestro-7dfb5b7d6b-qws2j", region="westus3", severity="warning"
  Description: Pod maestro/maestro-7dfb5b7d6b-qws2j has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Fired (not resolved)
  Started:     2026-10-16T16:13:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7660416-svc", component="kubernetes-infrastructure", namespace="sessiongate", pod="sessiongate-77fbb4f5f5-gdqfm", region="westus3", severity="warning"
  Description: Pod sessiongate/sessiongate-77fbb4f5f5-gdqfm has been in a non-ready state for longer than 5 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T15:54:14Z
  Ended:       2026-10-16T16:02:12Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j7660416-mgmt-1", component="monitoring-infrastructure", region="westus3", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T15:55:11Z
  Ended:       2026-10-16T16:02:12Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j7660416-mgmt-2", component="monitoring-infrastructure", region="westus3", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2a4a4982c560>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T14:33:12.4130712Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2a4a4982c540>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2a4a477c0190>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2a4a4982c520>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2a4a49889830>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2a4a4982c560>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T14:33:12.4130712Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2a4a4982c540>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2a4a477c0190>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2a4a4982c520>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2a4a49889830>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:467]: Timed out after 600.003s.
Certificate should be issued by Let's Encrypt staging via ACME DNS-01
Expected success, but got an error:
    <*errors.errorString | 0x23efaa396df0>: 
    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist
    {
        s: "certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:467]: Timed out after 600.003s.
Certificate should be issued by Let's Encrypt staging via ACME DNS-01
Expected success, but got an error:
    <*errors.errorString | 0x23efaa396df0>: 
    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist
    {
        s: "certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist",
    }

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T14:54:18Z
  Ended:       2026-10-16T15:01:19Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j3268992-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x281ad7cc620>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T14:22:29.6817536Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x281ad7cc600>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x281ad410320>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x281ad7cc5e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x281ae968cc0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x281ad7cc620>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T14:22:29.6817536Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x281ad7cc600>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x281ad410320>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x281ad7cc5e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x281ae968cc0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

time=2026-10-16T14:04:27.353Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description="Step cluster-create\n  Kind: Shell\n  Command: ./aks-cluster-create\n"
time=2026-10-16T14:34:26.879Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err="stamp 1: error running Shell Step, failed to execute shell command: {\"time\":\"2026-10-16T14:04:27.363830736Z\",\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j8714496-mgmt-1\",\"clusterName\":\"ci01-j8714496-mgmt-1\",\"region\":\"westus3\"}\n{\"time\":\"2026-10-16T14:04:29.161200885Z\",\"level\":\"INFO\",\"msg\":\"configured pools\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j8714496-mgmt-1\",\"clusterName\":\"ci01-j8714496-mgmt-1\",\"region\":\"westus3\",\"system\":1,\"worker\":3,\"infra\":2}\n{\"time\":\"2026-10-16T14:04:29.579255531Z\",\"level\":\"INFO\",\"msg\":\"deploying configured resources\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j8714496-mgmt-1\",\"clusterName\":\"ci01-j8714496-mgmt-1\",\"region\":\"westus3\",\"deployment\":\"aks-cluster-create\",\"poolCount\":6,\"clusterFields\":null}\n signal: killed"

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-16T14:26:06Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j8714496-mgmt-2", component="monitoring-infrastructure", region="westus3", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x102cc34803e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T14:12:28.9453936Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x102cc34803c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x102cc29781e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x102cc34803a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x102cc44c70b0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x102cc34803e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T14:12:28.9453936Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x102cc34803c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x102cc29781e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x102cc34803a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x102cc44c70b0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x340dc62fa018>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-dsg52jhcqb45" to finish creating: GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6",
      "name": "baecf025-dfba-4e06-b1e1-28ab3b4cbfe6",
      "status": "Failed",
      "startTime": "2026-10-16T14:26:03.913560661Z",
      "endTime": "2026-10-16T14:45:08.626305681Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6
            correlation ID: 85e240f2-d0e4-4c29-bd30-d4d4327881c2
            client request ID: d5c7b389-b2c6-4aba-9dfa-9055479c941f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x340dc62fa018>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-dsg52jhcqb45" to finish creating: GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6",
      "name": "baecf025-dfba-4e06-b1e1-28ab3b4cbfe6",
      "status": "Failed",
      "startTime": "2026-10-16T14:26:03.913560661Z",
      "endTime": "2026-10-16T14:45:08.626305681Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6
            correlation ID: 85e240f2-d0e4-4c29-bd30-d4d4327881c2
            client request ID: d5c7b389-b2c6-4aba-9dfa-9055479c941f
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster "cilium-cl"
Unexpected error:
    <*errors.joinError | 0x2dbe92ada858>: 
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cl" in resourcegroup="cni-cilium-t6jslt7lwjkt" to finish creating: GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69",
      "name": "88e58151-a475-4725-8e82-2d51b0f7fb69",
      "status": "Failed",
      "startTime": "2026-10-16T13:50:29.440992073Z",
      "endTime": "2026-10-16T14:09:32.07987309Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine cilium-cl-cilium-np-5p4h6-brpt4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-5p4h6-brpt4 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cl-cilium-np-5p4h6-brpt4: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-t6jslt7lwjkt--managed/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/availabilitySets/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after a... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69
            correlation ID: 054f0541-4905-4acf-9baf-353f2e747fdd
            client request ID: 320b7448-1575-4fdb-9611-9bf748713191
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster "cilium-cl"
Unexpected error:
    <*errors.joinError | 0x2dbe92ada858>: 
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cl" in resourcegroup="cni-cilium-t6jslt7lwjkt" to finish creating: GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69",
      "name": "88e58151-a475-4725-8e82-2d51b0f7fb69",
      "status": "Failed",
      "startTime": "2026-10-16T13:50:29.440992073Z",
      "endTime": "2026-10-16T14:09:32.07987309Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine cilium-cl-cilium-np-5p4h6-brpt4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-5p4h6-brpt4 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cl-cilium-np-5p4h6-brpt4: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-t6jslt7lwjkt--managed/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/availabilitySets/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after a... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69
            correlation ID: 054f0541-4905-4acf-9baf-353f2e747fdd
            client request ID: 320b7448-1575-4fdb-9611-9bf748713191
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x19d0afcab040>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T13:39:56.2350122Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x19d0afcab020>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x19d0af322be0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x19d0afcab000>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x19d0b1181830>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x19d0afcab040>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T13:39:56.2350122Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x19d0afcab020>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x19d0af322be0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x19d0afcab000>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x19d0b1181830>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.003s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x26aec5b792f0>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106->172.30.0.10:53: read: connection refused",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.003s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x26aec5b792f0>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106->172.30.0.10:53: read: connection refused",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity
Unexpected error:
    <*exported.ResponseError | 0x31fef30d4a50>: 
    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
    --------------------------------------------------------------------------------
    RESPONSE 429: 429 Too Many Requests
    ERROR CODE: 429
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "429",
        "message": "Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
            correlation ID: 63080195-ea99-42b1-8144-1ce2de77a2d8
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity
Unexpected error:
    <*exported.ResponseError | 0x31fef30d4a50>: 
    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
    --------------------------------------------------------------------------------
    RESPONSE 429: 429 Too Many Requests
    ERROR CODE: 429
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "429",
        "message": "Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service
            correlation ID: 63080195-ea99-42b1-8144-1ce2de77a2d8
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x33e95d68a9c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T13:12:53.4717042Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x33e95d68a9a0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x33e95c4f2d20>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x33e95d68a980>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x33e95d14a480>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x33e95d68a9c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T13:12:53.4717042Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x33e95d68a9a0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x33e95c4f2d20>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x33e95d68a980>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x33e95d14a480>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x239a48fa84c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x239a4a108640>{
            s: "dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x239a48fa84c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x239a4a108640>{
            s: "dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-r564c69p7xg4/cluster-candidate-5-1-bphh8v should provision
Unexpected error:
    <*fmt.wrapError | 0x25f3994d0160>: 
    failed to create HCP cluster cluster-candidate-5-1-bphh8v: failed waiting for cluster="cluster-candidate-5-1-bphh8v" in resourcegroup="rg-candidate-5-1-r564c69p7xg4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf",
      "name": "b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:46.450911767Z",
      "endTime": "2026-10-16T11:47:49.840205163Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf
            correlation ID: 9721cca3-00ee-40da-a65c-4dbeafe27604
            client request ID: 5c89be69-937f-4b9c-b3fe-2ac7fb2f335b
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-r564c69p7xg4/cluster-candidate-5-1-bphh8v should provision
Unexpected error:
    <*fmt.wrapError | 0x25f3994d0160>: 
    failed to create HCP cluster cluster-candidate-5-1-bphh8v: failed waiting for cluster="cluster-candidate-5-1-bphh8v" in resourcegroup="rg-candidate-5-1-r564c69p7xg4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf",
      "name": "b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:46.450911767Z",
      "endTime": "2026-10-16T11:47:49.840205163Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf
            correlation ID: 9721cca3-00ee-40da-a65c-4dbeafe27604
            client request ID: 5c89be69-937f-4b9c-b3fe-2ac7fb2f335b
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster "sre-hcp-cluster"
Unexpected error:
    <*fmt.wrapError | 0x267b45e33940>: 
    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster="sre-hcp-cluster" in resourcegroup="admin-api-breakglass-wg6vfj68ppbj" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3",
      "name": "7582db6b-1ec3-41f5-ae34-034a5f0015a3",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.316842517Z",
      "endTime": "2026-10-16T11:47:34.729108911Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3
            correlation ID: 672d62e4-3264-40b7-a323-c9f0c7db8552
            client request ID: 45b671db-f5be-402f-845b-ad031d68dc35
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster "sre-hcp-cluster"
Unexpected error:
    <*fmt.wrapError | 0x267b45e33940>: 
    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster="sre-hcp-cluster" in resourcegroup="admin-api-breakglass-wg6vfj68ppbj" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3",
      "name": "7582db6b-1ec3-41f5-ae34-034a5f0015a3",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.316842517Z",
      "endTime": "2026-10-16T11:47:34.729108911Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3
            correlation ID: 672d62e4-3264-40b7-a323-c9f0c7db8552
            client request ID: 45b671db-f5be-402f-845b-ad031d68dc35
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-22-zqjt7w" with version 4.22.17 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0xef740c20200>: 
    failed to create HCP cluster cluster-zstream-4-22-zqjt7w: failed waiting for cluster="cluster-zstream-4-22-zqjt7w" in resourcegroup="rg-zstream-upgrade-4-22-tgrmbldgf24k" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852",
      "name": "26a69e16-4222-4373-86da-5d09a67bc852",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:45.142598291Z",
      "endTime": "2026-10-16T11:47:49.837183985Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852
            correlation ID: 7f2b09a0-342f-4567-b3e6-0d9f66f0004d
            client request ID: 1f5c976b-37b6-4713-a33e-c70d3f34ed24
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-22-zqjt7w" with version 4.22.17 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0xef740c20200>: 
    failed to create HCP cluster cluster-zstream-4-22-zqjt7w: failed waiting for cluster="cluster-zstream-4-22-zqjt7w" in resourcegroup="rg-zstream-upgrade-4-22-tgrmbldgf24k" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852",
      "name": "26a69e16-4222-4373-86da-5d09a67bc852",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:45.142598291Z",
      "endTime": "2026-10-16T11:47:49.837183985Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852
            correlation ID: 7f2b09a0-342f-4567-b3e6-0d9f66f0004d
            client request ID: 1f5c976b-37b6-4713-a33e-c70d3f34ed24
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x302ea639cb20>: 
    failed to create HCP cluster np-dg-minor-84pjd6: failed waiting for cluster="np-dg-minor-84pjd6" in resourcegroup="rg-np-dg-minor-84pjd6-pxcbzv9wv7lk" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a",
      "name": "28010958-6488-4e47-a97f-3c980e40e80a",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.456420902Z",
      "endTime": "2026-10-16T12:43:31.119999655Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a
            correlation ID: a869227a-6203-4efd-a1fa-22d66ad064bb
            client request ID: dd4587bf-6acc-4c74-9458-b60c41fea5b3
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x302ea639cb20>: 
    failed to create HCP cluster np-dg-minor-84pjd6: failed waiting for cluster="np-dg-minor-84pjd6" in resourcegroup="rg-np-dg-minor-84pjd6-pxcbzv9wv7lk" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a",
      "name": "28010958-6488-4e47-a97f-3c980e40e80a",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.456420902Z",
      "endTime": "2026-10-16T12:43:31.119999655Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a
            correlation ID: a869227a-6203-4efd-a1fa-22d66ad064bb
            client request ID: dd4587bf-6acc-4c74-9458-b60c41fea5b3
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-qxqtw472st5s/cluster-candidate-4-22-kpqx7c should provision
Unexpected error:
    <*fmt.wrapError | 0x2ec0d0beadc0>: 
    failed to create HCP cluster cluster-candidate-4-22-kpqx7c: failed waiting for cluster="cluster-candidate-4-22-kpqx7c" in resourcegroup="rg-candidate-4-22-qxqtw472st5s" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f",
      "name": "37bf4b59-12e7-4663-92b5-2fa49b9e914f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.404569434Z",
      "endTime": "2026-10-16T11:47:34.737960425Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f
            correlation ID: 26fc1ea5-6812-46a0-ab6d-df6b6366a988
            client request ID: 4115d0b3-eb51-4b91-b3f2-2e25d96cfb16
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-qxqtw472st5s/cluster-candidate-4-22-kpqx7c should provision
Unexpected error:
    <*fmt.wrapError | 0x2ec0d0beadc0>: 
    failed to create HCP cluster cluster-candidate-4-22-kpqx7c: failed waiting for cluster="cluster-candidate-4-22-kpqx7c" in resourcegroup="rg-candidate-4-22-qxqtw472st5s" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f",
      "name": "37bf4b59-12e7-4663-92b5-2fa49b9e914f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.404569434Z",
      "endTime": "2026-10-16T11:47:34.737960425Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f
            correlation ID: 26fc1ea5-6812-46a0-ab6d-df6b6366a988
            client request ID: 4115d0b3-eb51-4b91-b3f2-2e25d96cfb16
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors
Unexpected error:
    <*fmt.wrapError | 0x1a9b1c8b6ba0>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster="idms-e2e-hcp-cluster" in resourcegroup="idms-csxwdq5kg9sx" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6",
      "name": "1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:40.61907571Z",
      "endTime": "2026-10-16T11:47:49.830293866Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6
            correlation ID: a282e9f7-9468-4fa7-aefb-7fc6d52c0c49
            client request ID: 0fa2af39-92d5-4643-befd-c770b11ab6af
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors
Unexpected error:
    <*fmt.wrapError | 0x1a9b1c8b6ba0>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster="idms-e2e-hcp-cluster" in resourcegroup="idms-csxwdq5kg9sx" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6",
      "name": "1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:40.61907571Z",
      "endTime": "2026-10-16T11:47:49.830293866Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6
            correlation ID: a282e9f7-9468-4fa7-aefb-7fc6d52c0c49
            client request ID: 0fa2af39-92d5-4643-befd-c770b11ab6af
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-23-rcc6r4" with version 4.23.0-ec.0 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x37d362890860>: 
    failed to create HCP cluster cluster-zstream-4-23-rcc6r4: failed waiting for cluster="cluster-zstream-4-23-rcc6r4" in resourcegroup="rg-zstream-upgrade-4-23-q79v76gpnl59" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6",
      "name": "ab63a679-4670-419c-aaa4-e0251814e5e6",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.848366298Z",
      "endTime": "2026-10-16T11:47:34.735398098Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6
            correlation ID: 7e0a0830-9f92-4675-99de-dfd56a431231
            client request ID: 391dd89d-6617-4212-aab7-a94d3c4c617f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-23-rcc6r4" with version 4.23.0-ec.0 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x37d362890860>: 
    failed to create HCP cluster cluster-zstream-4-23-rcc6r4: failed waiting for cluster="cluster-zstream-4-23-rcc6r4" in resourcegroup="rg-zstream-upgrade-4-23-q79v76gpnl59" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6",
      "name": "ab63a679-4670-419c-aaa4-e0251814e5e6",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.848366298Z",
      "endTime": "2026-10-16T11:47:34.735398098Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6
            correlation ID: 7e0a0830-9f92-4675-99de-dfd56a431231
            client request ID: 391dd89d-6617-4212-aab7-a94d3c4c617f
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-5-0-9pv22x" at install version 4.22
Unexpected error:
    <*fmt.wrapError | 0x2e350f5c8100>: 
    failed to create HCP cluster cp-ystream-upgrade-5-0-9pv22x: failed waiting for cluster="cp-ystream-upgrade-5-0-9pv22x" in resourcegroup="rg-cp-ystream-upgrade-5-0-xm47zptwcsjh" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead",
      "name": "cbc721e9-e4bf-4464-8b12-84eda9921ead",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:43.593514078Z",
      "endTime": "2026-10-16T11:47:49.853897009Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead
            correlation ID: d3fa4fbf-6b99-46ed-95f4-434c377ebaea
            client request ID: 4804a3ed-75bb-478f-aa09-5c0568252a82
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-5-0-9pv22x" at install version 4.22
Unexpected error:
    <*fmt.wrapError | 0x2e350f5c8100>: 
    failed to create HCP cluster cp-ystream-upgrade-5-0-9pv22x: failed waiting for cluster="cp-ystream-upgrade-5-0-9pv22x" in resourcegroup="rg-cp-ystream-upgrade-5-0-xm47zptwcsjh" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead",
      "name": "cbc721e9-e4bf-4464-8b12-84eda9921ead",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:43.593514078Z",
      "endTime": "2026-10-16T11:47:49.853897009Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead
            correlation ID: d3fa4fbf-6b99-46ed-95f4-434c377ebaea
            client request ID: 4804a3ed-75bb-478f-aa09-5c0568252a82
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-9jbnpswbqhnc/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-9jbnpswbqhnc/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit
Unexpected error:
    <*fmt.wrapError | 0x3cd963448a00>: 
    failed to create HCP cluster node-limit-cluster: failed waiting for cluster="node-limit-cluster" in resourcegroup="node-limit-test-g99wvvnb2gqt" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23",
      "name": "feb6dbe3-4518-4432-8ee1-e82bfe257e23",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:40.122769052Z",
      "endTime": "2026-10-16T11:47:49.816923189Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23
            correlation ID: 6e26fa23-6c8f-48c7-84e4-93c4480408ac
            client request ID: 0de8699c-e23a-4388-aef8-31bf31c0384d
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit
Unexpected error:
    <*fmt.wrapError | 0x3cd963448a00>: 
    failed to create HCP cluster node-limit-cluster: failed waiting for cluster="node-limit-cluster" in resourcegroup="node-limit-test-g99wvvnb2gqt" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23",
      "name": "feb6dbe3-4518-4432-8ee1-e82bfe257e23",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:40.122769052Z",
      "endTime": "2026-10-16T11:47:49.816923189Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23
            correlation ID: 6e26fa23-6c8f-48c7-84e4-93c4480408ac
            client request ID: 0de8699c-e23a-4388-aef8-31bf31c0384d
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints
Unexpected error:
    <*fmt.wrapError | 0x3d54a9ec8640>: 
    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster="cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-gpfqm2n76z2f" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643",
      "name": "9319e8d3-f312-4003-b90c-21f9495a0643",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:40.951220038Z",
      "endTime": "2026-10-16T11:47:49.835941737Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643
            correlation ID: 7ab67cc1-868f-400c-b479-651074342b57
            client request ID: f0c8515f-2da0-4d17-b772-fbcda0b27f98
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints
Unexpected error:
    <*fmt.wrapError | 0x3d54a9ec8640>: 
    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster="cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-gpfqm2n76z2f" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643",
      "name": "9319e8d3-f312-4003-b90c-21f9495a0643",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:40.951220038Z",
      "endTime": "2026-10-16T11:47:49.835941737Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643
            correlation ID: 7ab67cc1-868f-400c-b479-651074342b57
            client request ID: f0c8515f-2da0-4d17-b772-fbcda0b27f98
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster "hcp-cluster-np-128"
Unexpected error:
    <*fmt.wrapError | 0x30530c2049c0>: 
    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster="hcp-cluster-np-128" in resourcegroup="clusternp128-pq2pqkkk7kkv" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1",
      "name": "e1c4fc61-b13a-4949-9525-c8e2995366d1",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.276521952Z",
      "endTime": "2026-10-16T11:47:34.729548419Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1
            correlation ID: cc4b631c-4cdd-46df-8676-8256af846370
            client request ID: b0546f64-7657-4fa7-8053-0510bf32e452
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster "hcp-cluster-np-128"
Unexpected error:
    <*fmt.wrapError | 0x30530c2049c0>: 
    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster="hcp-cluster-np-128" in resourcegroup="clusternp128-pq2pqkkk7kkv" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1",
      "name": "e1c4fc61-b13a-4949-9525-c8e2995366d1",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.276521952Z",
      "endTime": "2026-10-16T11:47:34.729548419Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1
            correlation ID: cc4b631c-4cdd-46df-8676-8256af846370
            client request ID: b0546f64-7657-4fa7-8053-0510bf32e452
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-nlkpgh277c95/cluster-candidate-4-23-n82d96 should provision
Unexpected error:
    <*fmt.wrapError | 0x1649075acb60>: 
    failed to create HCP cluster cluster-candidate-4-23-n82d96: failed waiting for cluster="cluster-candidate-4-23-n82d96" in resourcegroup="rg-candidate-4-23-nlkpgh277c95" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3",
      "name": "27de5df4-c265-4de7-97d7-ba19926470a3",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.252038061Z",
      "endTime": "2026-10-16T11:47:35.485949636Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3
            correlation ID: 375617cf-8ae4-4fb0-8226-a993b64c2efa
            client request ID: 1d6dd6d4-ca93-447b-9e62-a692550d6007
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-nlkpgh277c95/cluster-candidate-4-23-n82d96 should provision
Unexpected error:
    <*fmt.wrapError | 0x1649075acb60>: 
    failed to create HCP cluster cluster-candidate-4-23-n82d96: failed waiting for cluster="cluster-candidate-4-23-n82d96" in resourcegroup="rg-candidate-4-23-nlkpgh277c95" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3",
      "name": "27de5df4-c265-4de7-97d7-ba19926470a3",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.252038061Z",
      "endTime": "2026-10-16T11:47:35.485949636Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3
            correlation ID: 375617cf-8ae4-4fb0-8226-a993b64c2efa
            client request ID: 1d6dd6d4-ca93-447b-9e62-a692550d6007
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster "cidr-connectivity-test" with authorized CIDRs
Unexpected error:
    <*fmt.wrapError | 0xc7fcee78400>: 
    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-xkdmlz6n9jw9" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035",
      "name": "9437713a-127b-49bb-b254-8e11c4cea035",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:30.237979438Z",
      "endTime": "2026-10-16T11:48:39.854496595Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035
            correlation ID: c47ad550-f9bc-4b54-9fb4-b9c071da2e2e
            client request ID: 7908e69f-5b28-487e-b354-e80c3f6a203f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster "cidr-connectivity-test" with authorized CIDRs
Unexpected error:
    <*fmt.wrapError | 0xc7fcee78400>: 
    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-xkdmlz6n9jw9" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035",
      "name": "9437713a-127b-49bb-b254-8e11c4cea035",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:30.237979438Z",
      "endTime": "2026-10-16T11:48:39.854496595Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035
            correlation ID: c47ad550-f9bc-4b54-9fb4-b9c071da2e2e
            client request ID: 7908e69f-5b28-487e-b354-e80c3f6a203f
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster "fips-enabled-cluster" with cryptoRestrictions set to FIPS
Unexpected error:
    <*fmt.wrapError | 0x2618dbece140>: 
    failed waiting for cluster="fips-enabled-cluster" in resourcegroup="fips-enabled-j5vlvmgjsqjz" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645",
      "name": "a389c8be-8768-4995-ad5b-c321a6398645",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:56.786424151Z",
      "endTime": "2026-10-16T11:47:59.869771907Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645
            correlation ID: a9be42e7-990f-46ed-82b2-614c0f5c93e4
            client request ID: f1d3955d-3265-447b-966c-cfe8c54031e0
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster "fips-enabled-cluster" with cryptoRestrictions set to FIPS
Unexpected error:
    <*fmt.wrapError | 0x2618dbece140>: 
    failed waiting for cluster="fips-enabled-cluster" in resourcegroup="fips-enabled-j5vlvmgjsqjz" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645",
      "name": "a389c8be-8768-4995-ad5b-c321a6398645",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:56.786424151Z",
      "endTime": "2026-10-16T11:47:59.869771907Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645
            correlation ID: a9be42e7-990f-46ed-82b2-614c0f5c93e4
            client request ID: f1d3955d-3265-447b-966c-cfe8c54031e0
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster
Unexpected error:
    <*fmt.wrapError | 0xeb24e5fc1c0>: 
    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster="negative-tests-cluster" in resourcegroup="rg-negative-tests-s7wwcnxg7nr6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff",
      "name": "bddfb7c0-8782-439a-a29c-5d984838d4ff",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:30.158724197Z",
      "endTime": "2026-10-16T12:43:31.12042993Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff
            correlation ID: 76f022c5-91e3-4c33-8b07-8710b12f4b32
            client request ID: 6e812c92-cfd2-484b-9f9d-c35d3bef9df7
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster
Unexpected error:
    <*fmt.wrapError | 0xeb24e5fc1c0>: 
    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster="negative-tests-cluster" in resourcegroup="rg-negative-tests-s7wwcnxg7nr6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff",
      "name": "bddfb7c0-8782-439a-a29c-5d984838d4ff",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:30.158724197Z",
      "endTime": "2026-10-16T12:43:31.12042993Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff
            correlation ID: 76f022c5-91e3-4c33-8b07-8710b12f4b32
            client request ID: 6e812c92-cfd2-484b-9f9d-c35d3bef9df7
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-k8896m6vhvph/cluster-candidate-5-0-svffqc should provision
Unexpected error:
    <*fmt.wrapError | 0x4408644c620>: 
    failed to create HCP cluster cluster-candidate-5-0-svffqc: failed waiting for cluster="cluster-candidate-5-0-svffqc" in resourcegroup="rg-candidate-5-0-k8896m6vhvph" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc",
      "name": "f521b84d-3184-4232-9fe7-cd33b6490fcc",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.238638011Z",
      "endTime": "2026-10-16T11:47:34.761578844Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc
            correlation ID: edad4ec0-1e19-4f0a-84a2-f8ef2fb08495
            client request ID: 6ec90bea-9fe0-4677-b51f-03ee25f8acb9
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-k8896m6vhvph/cluster-candidate-5-0-svffqc should provision
Unexpected error:
    <*fmt.wrapError | 0x4408644c620>: 
    failed to create HCP cluster cluster-candidate-5-0-svffqc: failed waiting for cluster="cluster-candidate-5-0-svffqc" in resourcegroup="rg-candidate-5-0-k8896m6vhvph" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc",
      "name": "f521b84d-3184-4232-9fe7-cd33b6490fcc",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.238638011Z",
      "endTime": "2026-10-16T11:47:34.761578844Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc
            correlation ID: edad4ec0-1e19-4f0a-84a2-f8ef2fb08495
            client request ID: 6ec90bea-9fe0-4677-b51f-03ee25f8acb9
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kmzj2j with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x163f9a6e2b60>: 
    failed to create HCP cluster np-version-upgrade-cluster-kmzj2j: failed waiting for cluster="np-version-upgrade-cluster-kmzj2j" in resourcegroup="rg-np-version-upgrade-kmzj2j-88dfmkbc2kcf" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff",
      "name": "3112585c-f06a-4eed-8bc2-c5766d255eff",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:30.034396275Z",
      "endTime": "2026-10-16T12:43:31.11907947Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff
            correlation ID: 61a6bffb-48df-441c-9977-a498349294c0
            client request ID: 92258152-5b22-402e-88a2-d9f0a0d3f850
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kmzj2j with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x163f9a6e2b60>: 
    failed to create HCP cluster np-version-upgrade-cluster-kmzj2j: failed waiting for cluster="np-version-upgrade-cluster-kmzj2j" in resourcegroup="rg-np-version-upgrade-kmzj2j-88dfmkbc2kcf" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff",
      "name": "3112585c-f06a-4eed-8bc2-c5766d255eff",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:30.034396275Z",
      "endTime": "2026-10-16T12:43:31.11907947Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff
            correlation ID: 61a6bffb-48df-441c-9977-a498349294c0
            client request ID: 92258152-5b22-402e-88a2-d9f0a0d3f850
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed
fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x57a86294180>: 
    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster="np-delete-hcp-cluster" in resourcegroup="nodepool-delete-wqj2khzntl8h" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a",
      "name": "a00835a9-2946-492c-83cf-4feb60e97d6a",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:41.970124073Z",
      "endTime": "2026-10-16T11:47:49.845444522Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a
            correlation ID: b829a774-d420-42d1-910f-558c24c0e81b
            client request ID: fa3b6c1d-d6ce-4bc1-818f-7e083d5113d0
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x57a86294180>: 
    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster="np-delete-hcp-cluster" in resourcegroup="nodepool-delete-wqj2khzntl8h" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a",
      "name": "a00835a9-2946-492c-83cf-4feb60e97d6a",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:41.970124073Z",
      "endTime": "2026-10-16T11:47:49.845444522Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a
            correlation ID: b829a774-d420-42d1-910f-558c24c0e81b
            client request ID: fa3b6c1d-d6ce-4bc1-818f-7e083d5113d0
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster "cilium-cl" without CNI
Unexpected error:
    <*fmt.wrapError | 0x3e5b5b4ea680>: 
    failed to create HCP cluster cilium-cl: failed waiting for cluster="cilium-cl" in resourcegroup="cni-cilium-pqhbfg7fld4c" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f",
      "name": "934784ee-76b6-4805-b96a-804962fd4f8f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.866312322Z",
      "endTime": "2026-10-16T11:47:15.217711166Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f
            correlation ID: 0f7d9c8a-342c-410a-bac7-fcbf38991b8b
            client request ID: e542720e-5ece-4d3c-a284-7d8bdebc13b0
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster "cilium-cl" without CNI
Unexpected error:
    <*fmt.wrapError | 0x3e5b5b4ea680>: 
    failed to create HCP cluster cilium-cl: failed waiting for cluster="cilium-cl" in resourcegroup="cni-cilium-pqhbfg7fld4c" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f",
      "name": "934784ee-76b6-4805-b96a-804962fd4f8f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.866312322Z",
      "endTime": "2026-10-16T11:47:15.217711166Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f
            correlation ID: 0f7d9c8a-342c-410a-bac7-fcbf38991b8b
            client request ID: e542720e-5ece-4d3c-a284-7d8bdebc13b0
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled
Unexpected error:
    <*fmt.wrapError | 0xbcb4282a180>: 
    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster="disabled-image-registry-hcp-cluster" in resourcegroup="disabled-image-registry-x5pjtlmznn57" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f",
      "name": "6909c02f-59ad-446d-af3d-48126d54318f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:41.793037251Z",
      "endTime": "2026-10-16T11:47:49.840680018Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f
            correlation ID: 6249e6eb-e2f2-49a4-8b3d-fa8588c616f1
            client request ID: fcd7ba19-a355-4a1f-b414-c70f5d773a28
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled
Unexpected error:
    <*fmt.wrapError | 0xbcb4282a180>: 
    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster="disabled-image-registry-hcp-cluster" in resourcegroup="disabled-image-registry-x5pjtlmznn57" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f",
      "name": "6909c02f-59ad-446d-af3d-48126d54318f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:41.793037251Z",
      "endTime": "2026-10-16T11:47:49.840680018Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f
            correlation ID: 6249e6eb-e2f2-49a4-8b3d-fa8588c616f1
            client request ID: fcd7ba19-a355-4a1f-b414-c70f5d773a28
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-js4rfm" with version 4.20.42 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1a1f4bafcbc0>: 
    failed to create HCP cluster cluster-zstream-4-20-js4rfm: failed waiting for cluster="cluster-zstream-4-20-js4rfm" in resourcegroup="rg-zstream-upgrade-4-20-r2p2qj59lsd4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f",
      "name": "862c5703-2680-4bf6-9df4-c4097e46e34f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:46.295357285Z",
      "endTime": "2026-10-16T11:47:49.823361811Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f
            correlation ID: 38689985-8663-45c2-b6d3-3c921b0b9d8e
            client request ID: de817d4d-06c3-4aa8-aeb2-34912bfe8ec1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-js4rfm" with version 4.20.42 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1a1f4bafcbc0>: 
    failed to create HCP cluster cluster-zstream-4-20-js4rfm: failed waiting for cluster="cluster-zstream-4-20-js4rfm" in resourcegroup="rg-zstream-upgrade-4-20-r2p2qj59lsd4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f",
      "name": "862c5703-2680-4bf6-9df4-c4097e46e34f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:46.295357285Z",
      "endTime": "2026-10-16T11:47:49.823361811Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f
            correlation ID: 38689985-8663-45c2-b6d3-3c921b0b9d8e
            client request ID: de817d4d-06c3-4aa8-aeb2-34912bfe8ec1
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster
Unexpected error:
    <*fmt.wrapError | 0x1edc8262aa20>: 
    failed to create HCP cluster basic-cluster: failed waiting for cluster="basic-cluster" in resourcegroup="rg-cluster-nsg-subnet-reuse-6p6j6qsmdwpj" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7",
      "name": "4a5c3106-eb2a-45eb-a602-c5ec51db82a7",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.696535718Z",
      "endTime": "2026-10-16T11:47:34.770284565Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7
            correlation ID: faeb3822-724b-4825-a824-edb8c1b845c1
            client request ID: 51d01eb4-edcc-4337-ad97-569b5b96ddb2
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster
Unexpected error:
    <*fmt.wrapError | 0x1edc8262aa20>: 
    failed to create HCP cluster basic-cluster: failed waiting for cluster="basic-cluster" in resourcegroup="rg-cluster-nsg-subnet-reuse-6p6j6qsmdwpj" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7",
      "name": "4a5c3106-eb2a-45eb-a602-c5ec51db82a7",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.696535718Z",
      "endTime": "2026-10-16T11:47:34.770284565Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7
            correlation ID: faeb3822-724b-4825-a824-edb8c1b845c1
            client request ID: 51d01eb4-edcc-4337-ad97-569b5b96ddb2
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:126]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0xaa8af49c580>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-zsmzg99h2cph" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6",
      "name": "a4151c0d-6fe7-46e7-ae06-85bca0bbfda6",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:28.3068615Z",
      "endTime": "2026-10-16T11:48:29.818041797Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6
            correlation ID: 5e6c1f91-f464-4ac3-8b66-a7151ccf033d
            client request ID: 1763d574-e139-47a7-a99c-8d752034afa4
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:126]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0xaa8af49c580>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-zsmzg99h2cph" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6",
      "name": "a4151c0d-6fe7-46e7-ae06-85bca0bbfda6",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:28.3068615Z",
      "endTime": "2026-10-16T11:48:29.818041797Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6
            correlation ID: 5e6c1f91-f464-4ac3-8b66-a7151ccf033d
            client request ID: 1763d574-e139-47a7-a99c-8d752034afa4
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster "private-ingress" with private ingress
Unexpected error:
    <*fmt.wrapError | 0x2cc9b8996a60>: 
    failed to create HCP cluster private-ingress: failed waiting for cluster="private-ingress" in resourcegroup="private-ingress-w5rdt4gkt25d" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b",
      "name": "1a2ea3fa-8f68-40b2-a65c-a717ffa2902b",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:36.212608814Z",
      "endTime": "2026-10-16T11:47:39.813722246Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b
            correlation ID: a01e533d-97b0-4d30-8c31-a1730fbb9c1b
            client request ID: dd83ab35-b0c7-4853-b865-900c9dc97b10
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster "private-ingress" with private ingress
Unexpected error:
    <*fmt.wrapError | 0x2cc9b8996a60>: 
    failed to create HCP cluster private-ingress: failed waiting for cluster="private-ingress" in resourcegroup="private-ingress-w5rdt4gkt25d" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b",
      "name": "1a2ea3fa-8f68-40b2-a65c-a717ffa2902b",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:36.212608814Z",
      "endTime": "2026-10-16T11:47:39.813722246Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b
            correlation ID: a01e533d-97b0-4d30-8c31-a1730fbb9c1b
            client request ID: dd83ab35-b0c7-4853-b865-900c9dc97b10
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-x5jpgh
Unexpected error:
    <*fmt.wrapError | 0x2ba9f4280b60>: 
    failed to create HCP cluster np-noedge-x5jpgh: failed waiting for cluster="np-noedge-x5jpgh" in resourcegroup="rg-np-noedge-x5jpgh-nmz7rtgztcdt" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2",
      "name": "a63b7a0e-c2ff-4ea5-ad50-a2392535dea2",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:28.980964675Z",
      "endTime": "2026-10-16T12:43:31.119616474Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2
            correlation ID: a6626179-2136-45bb-825a-5b925656e4cd
            client request ID: 28e2874d-4a4a-42c0-9201-66d388c353c7
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-x5jpgh
Unexpected error:
    <*fmt.wrapError | 0x2ba9f4280b60>: 
    failed to create HCP cluster np-noedge-x5jpgh: failed waiting for cluster="np-noedge-x5jpgh" in resourcegroup="rg-np-noedge-x5jpgh-nmz7rtgztcdt" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2",
      "name": "a63b7a0e-c2ff-4ea5-ad50-a2392535dea2",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:28.980964675Z",
      "endTime": "2026-10-16T12:43:31.119616474Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2
            correlation ID: a6626179-2136-45bb-825a-5b925656e4cd
            client request ID: 28e2874d-4a4a-42c0-9201-66d388c353c7
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster "autoscaling-hcp-cluster" with custom autoscaling
Unexpected error:
    <*fmt.wrapError | 0x349580025280>: 
    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster="autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-xbkhmb99h5bw" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132",
      "name": "56bd2ed4-8b90-4d1e-9b32-b3e975d36132",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.895598807Z",
      "endTime": "2026-10-16T11:47:14.891231174Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132
            correlation ID: 44a81805-e64c-4ba6-9409-983e8b170312
            client request ID: 0115f71b-c2d8-4f13-b17d-9fb8699799fd
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster "autoscaling-hcp-cluster" with custom autoscaling
Unexpected error:
    <*fmt.wrapError | 0x349580025280>: 
    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster="autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-xbkhmb99h5bw" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132",
      "name": "56bd2ed4-8b90-4d1e-9b32-b3e975d36132",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.895598807Z",
      "endTime": "2026-10-16T11:47:14.891231174Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132
            correlation ID: 44a81805-e64c-4ba6-9409-983e8b170312
            client request ID: 0115f71b-c2d8-4f13-b17d-9fb8699799fd
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x123425a4b400>: 
    failed to create HCP cluster np-downgrade-8lhgvh: failed waiting for cluster="np-downgrade-8lhgvh" in resourcegroup="rg-np-downgrade-8lhgvh-d55cjbncvrzk" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9",
      "name": "c6e38d64-f6cf-46fa-8335-12cd44b337a9",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:28.696987476Z",
      "endTime": "2026-10-16T12:43:31.118819677Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9
            correlation ID: 433e0cbf-f087-4fdc-b83e-56c9b436990e
            client request ID: a497577e-eb3d-4aa3-a690-f0283dedfcd8
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x123425a4b400>: 
    failed to create HCP cluster np-downgrade-8lhgvh: failed waiting for cluster="np-downgrade-8lhgvh" in resourcegroup="rg-np-downgrade-8lhgvh-d55cjbncvrzk" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9",
      "name": "c6e38d64-f6cf-46fa-8335-12cd44b337a9",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:28.696987476Z",
      "endTime": "2026-10-16T12:43:31.118819677Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9
            correlation ID: 433e0cbf-f087-4fdc-b83e-56c9b436990e
            client request ID: a497577e-eb3d-4aa3-a690-f0283dedfcd8
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster "ksm-hcp-metrics"
Unexpected error:
    <*fmt.wrapError | 0x32fe88162120>: 
    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster="ksm-hcp-metrics" in resourcegroup="hcp-metrics-pjd2p5pkfgjp" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee",
      "name": "3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:42.802740388Z",
      "endTime": "2026-10-16T11:47:49.838674727Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee
            correlation ID: 368abfde-4c93-4713-8780-51b84de0d719
            client request ID: c27bf18a-f74b-4aba-9e88-1de4311a6a7c
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster "ksm-hcp-metrics"
Unexpected error:
    <*fmt.wrapError | 0x32fe88162120>: 
    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster="ksm-hcp-metrics" in resourcegroup="hcp-metrics-pjd2p5pkfgjp" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee",
      "name": "3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:42.802740388Z",
      "endTime": "2026-10-16T11:47:49.838674727Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee
            correlation ID: 368abfde-4c93-4713-8780-51b84de0d719
            client request ID: c27bf18a-f74b-4aba-9e88-1de4311a6a7c
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test
Unexpected error:
    <*fmt.wrapError | 0x390e05a68de0>: 
    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster="patch-tags-cluster" in resourcegroup="patch-tags-xqknpzfzmr5w" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65",
      "name": "73d23223-112b-4b7f-ba15-3c3397a86d65",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.514720241Z",
      "endTime": "2026-10-16T11:47:24.729519268Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65
            correlation ID: 88facd09-aef2-4a7f-9c7e-db87f515aac5
            client request ID: 4c75b22c-aded-487f-a952-f949adbe49c6
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test
Unexpected error:
    <*fmt.wrapError | 0x390e05a68de0>: 
    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster="patch-tags-cluster" in resourcegroup="patch-tags-xqknpzfzmr5w" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65",
      "name": "73d23223-112b-4b7f-ba15-3c3397a86d65",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.514720241Z",
      "endTime": "2026-10-16T11:47:24.729519268Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65
            correlation ID: 88facd09-aef2-4a7f-9c7e-db87f515aac5
            client request ID: 4c75b22c-aded-487f-a952-f949adbe49c6
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.000s.
HCP resource requirements were not populated in time
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:
DataCurrent condition must be True
Expected
    <bool>: false
to be true
fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.000s.
HCP resource requirements were not populated in time
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:
DataCurrent condition must be True
Expected
    <bool>: false
to be true

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster "v20260901" via v20260901preview
Unexpected error:
    <*fmt.wrapError | 0xe1af249a880>: 
    failed to create HCP cluster v20260901: failed waiting for cluster="v20260901" in resourcegroup="v20260901preview-6jlkrz7lq599" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f",
      "name": "634b8f7c-5c70-49a7-b19e-48bd89488e6f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.833452215Z",
      "endTime": "2026-10-16T11:47:15.218786602Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f
            correlation ID: b56813d3-03b1-46c4-a01a-7aa714d3bb3b
            client request ID: 6bba57fa-9f7a-491b-9479-621cd45e5270
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster "v20260901" via v20260901preview
Unexpected error:
    <*fmt.wrapError | 0xe1af249a880>: 
    failed to create HCP cluster v20260901: failed waiting for cluster="v20260901" in resourcegroup="v20260901preview-6jlkrz7lq599" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f",
      "name": "634b8f7c-5c70-49a7-b19e-48bd89488e6f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.833452215Z",
      "endTime": "2026-10-16T11:47:15.218786602Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f
            correlation ID: b56813d3-03b1-46c4-a01a-7aa714d3bb3b
            client request ID: 6bba57fa-9f7a-491b-9479-621cd45e5270
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk
Unexpected error:
    <*fmt.wrapError | 0x26ae0b1187a0>: 
    failed to create HCP cluster ephemeral-disk: failed waiting for cluster="ephemeral-disk" in resourcegroup="ephemeral-osdisk-98xd6plvc54r" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a",
      "name": "ef85c04a-5d49-4fdd-934c-fb1c0925da5a",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:51.520890154Z",
      "endTime": "2026-10-16T11:47:59.828901888Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a
            correlation ID: 61b08897-2788-477c-9e28-af5643f8731a
            client request ID: f65210df-702d-4b5f-bf36-5cb3949c8b60
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk
Unexpected error:
    <*fmt.wrapError | 0x26ae0b1187a0>: 
    failed to create HCP cluster ephemeral-disk: failed waiting for cluster="ephemeral-disk" in resourcegroup="ephemeral-osdisk-98xd6plvc54r" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a",
      "name": "ef85c04a-5d49-4fdd-934c-fb1c0925da5a",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:51.520890154Z",
      "endTime": "2026-10-16T11:47:59.828901888Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a
            correlation ID: 61b08897-2788-477c-9e28-af5643f8731a
            client request ID: f65210df-702d-4b5f-bf36-5cb3949c8b60
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster "sre-version-pin-bktv28" at 4.21.37
Unexpected error:
    <*fmt.wrapError | 0x28142a660f40>: 
    failed to create HCP cluster sre-version-pin-bktv28: failed waiting for cluster="sre-version-pin-bktv28" in resourcegroup="admin-api-version-pin-nqqcn8wlvf5c" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031",
      "name": "b2b04de1-00d3-4f87-be7d-90b2c747d031",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:41.261950808Z",
      "endTime": "2026-10-16T11:47:49.830086043Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031
            correlation ID: c9fa32a4-a67f-4d74-87e4-0703a6dd15ef
            client request ID: 4b87fe24-0400-4a09-8cfa-9dbfa250ce81
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster "sre-version-pin-bktv28" at 4.21.37
Unexpected error:
    <*fmt.wrapError | 0x28142a660f40>: 
    failed to create HCP cluster sre-version-pin-bktv28: failed waiting for cluster="sre-version-pin-bktv28" in resourcegroup="admin-api-version-pin-nqqcn8wlvf5c" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031",
      "name": "b2b04de1-00d3-4f87-be7d-90b2c747d031",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:41.261950808Z",
      "endTime": "2026-10-16T11:47:49.830086043Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031
            correlation ID: c9fa32a4-a67f-4d74-87e4-0703a6dd15ef
            client request ID: 4b87fe24-0400-4a09-8cfa-9dbfa250ce81
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-g5rdgl7zwjl6/cluster-candidate-4-20-k9b9wf should provision
Unexpected error:
    <*fmt.wrapError | 0x18f77dd205c0>: 
    failed to create HCP cluster cluster-candidate-4-20-k9b9wf: failed waiting for cluster="cluster-candidate-4-20-k9b9wf" in resourcegroup="rg-candidate-4-20-g5rdgl7zwjl6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314",
      "name": "0984890f-0a19-47d9-9a1c-275d44c6e314",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:36.959680763Z",
      "endTime": "2026-10-16T11:47:39.816204138Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314
            correlation ID: c1d2f06b-ddfc-4fd2-8766-74fa80998c3b
            client request ID: ef5cac13-1a05-485d-b15a-d283225aac9d
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-g5rdgl7zwjl6/cluster-candidate-4-20-k9b9wf should provision
Unexpected error:
    <*fmt.wrapError | 0x18f77dd205c0>: 
    failed to create HCP cluster cluster-candidate-4-20-k9b9wf: failed waiting for cluster="cluster-candidate-4-20-k9b9wf" in resourcegroup="rg-candidate-4-20-g5rdgl7zwjl6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314",
      "name": "0984890f-0a19-47d9-9a1c-275d44c6e314",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:36.959680763Z",
      "endTime": "2026-10-16T11:47:39.816204138Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314
            correlation ID: c1d2f06b-ddfc-4fd2-8766-74fa80998c3b
            client request ID: ef5cac13-1a05-485d-b15a-d283225aac9d
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-23-8xw5fg" at install version 4.22
Unexpected error:
    <*fmt.wrapError | 0x370f8d648c60>: 
    failed to create HCP cluster cp-ystream-upgrade-4-23-8xw5fg: failed waiting for cluster="cp-ystream-upgrade-4-23-8xw5fg" in resourcegroup="rg-cp-ystream-upgrade-4-23-b5xx2gk9qwbp" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f",
      "name": "56bd1583-21e2-4843-b05f-31ac6826444f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:33.267640725Z",
      "endTime": "2026-10-16T11:47:35.486996122Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f
            correlation ID: 3c2cdb81-95c2-4c03-9b23-7f4468666f93
            client request ID: ca21a34b-5eb9-4e2d-9400-6431d58f4052
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-23-8xw5fg" at install version 4.22
Unexpected error:
    <*fmt.wrapError | 0x370f8d648c60>: 
    failed to create HCP cluster cp-ystream-upgrade-4-23-8xw5fg: failed waiting for cluster="cp-ystream-upgrade-4-23-8xw5fg" in resourcegroup="rg-cp-ystream-upgrade-4-23-b5xx2gk9qwbp" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f",
      "name": "56bd1583-21e2-4843-b05f-31ac6826444f",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:33.267640725Z",
      "endTime": "2026-10-16T11:47:35.486996122Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f
            correlation ID: 3c2cdb81-95c2-4c03-9b23-7f4468666f93
            client request ID: ca21a34b-5eb9-4e2d-9400-6431d58f4052
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-zmn6qg at version 4.22.18
Unexpected error:
    <*fmt.wrapError | 0x8551e630ea0>: 
    failed to create HCP cluster np-skip-minor-zmn6qg: failed waiting for cluster="np-skip-minor-zmn6qg" in resourcegroup="rg-np-skip-minor-zmn6qg-zq4qtnxjmlj6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e",
      "name": "99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.343845082Z",
      "endTime": "2026-10-16T12:43:31.120076578Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e
            correlation ID: ae7f1a1f-77db-45da-834f-d7f445c5f80e
            client request ID: abc08cab-9ae2-473b-a212-d5cf27cc3368
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-zmn6qg at version 4.22.18
Unexpected error:
    <*fmt.wrapError | 0x8551e630ea0>: 
    failed to create HCP cluster np-skip-minor-zmn6qg: failed waiting for cluster="np-skip-minor-zmn6qg" in resourcegroup="rg-np-skip-minor-zmn6qg-zq4qtnxjmlj6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e",
      "name": "99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.343845082Z",
      "endTime": "2026-10-16T12:43:31.120076578Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e
            correlation ID: ae7f1a1f-77db-45da-834f-d7f445c5f80e
            client request ID: abc08cab-9ae2-473b-a212-d5cf27cc3368
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster "sre-hcp-cluster-sc" for serial console test
Unexpected error:
    <*fmt.wrapError | 0xb4e8f680740>: 
    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster="sre-hcp-cluster-sc" in resourcegroup="admin-api-serialconsole-6jkf7zpl5td4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1",
      "name": "19c9efd8-d627-4375-9ef4-9babf234f8d1",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.041145923Z",
      "endTime": "2026-10-16T11:47:14.862669197Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1
            correlation ID: 9c949c6a-68de-4443-a83a-d888a74b990c
            client request ID: 4d47ea32-d094-469c-af18-9dd47dff7a28
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster "sre-hcp-cluster-sc" for serial console test
Unexpected error:
    <*fmt.wrapError | 0xb4e8f680740>: 
    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster="sre-hcp-cluster-sc" in resourcegroup="admin-api-serialconsole-6jkf7zpl5td4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1",
      "name": "19c9efd8-d627-4375-9ef4-9babf234f8d1",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.041145923Z",
      "endTime": "2026-10-16T11:47:14.862669197Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1
            correlation ID: 9c949c6a-68de-4443-a83a-d888a74b990c
            client request ID: 4d47ea32-d094-469c-af18-9dd47dff7a28
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.000s.
scheduling data was not populated in time
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:
stamp 1 scheduling requested CPU must be positive
Expected
    <bool>: true
to be false
fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.000s.
scheduling data was not populated in time
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:
stamp 1 scheduling requested CPU must be positive
Expected
    <bool>: true
to be false

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-m2vs55vfxjl6/cluster-candidate-4-21-tkgb5h should provision
Unexpected error:
    <*fmt.wrapError | 0x3e0a8449c940>: 
    failed to create HCP cluster cluster-candidate-4-21-tkgb5h: failed waiting for cluster="cluster-candidate-4-21-tkgb5h" in resourcegroup="rg-candidate-4-21-m2vs55vfxjl6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf",
      "name": "62fb450c-e7f1-4190-bb7f-226b4ee61acf",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.933025884Z",
      "endTime": "2026-10-16T11:47:15.220182174Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf
            correlation ID: 47d3a69f-84e1-4042-80f0-1c8fbfff617f
            client request ID: f0b59b24-6a55-4893-8114-1d2b9df17303
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-m2vs55vfxjl6/cluster-candidate-4-21-tkgb5h should provision
Unexpected error:
    <*fmt.wrapError | 0x3e0a8449c940>: 
    failed to create HCP cluster cluster-candidate-4-21-tkgb5h: failed waiting for cluster="cluster-candidate-4-21-tkgb5h" in resourcegroup="rg-candidate-4-21-m2vs55vfxjl6" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf",
      "name": "62fb450c-e7f1-4190-bb7f-226b4ee61acf",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.933025884Z",
      "endTime": "2026-10-16T11:47:15.220182174Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf
            correlation ID: 47d3a69f-84e1-4042-80f0-1c8fbfff617f
            client request ID: f0b59b24-6a55-4893-8114-1d2b9df17303
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt
Unexpected error:
    <*fmt.wrapError | 0x3829cd510c40>: 
    failed to create HCP cluster des-encrypt: failed waiting for cluster="des-encrypt" in resourcegroup="des-encrypt-4gqdqvkfqdht" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3",
      "name": "43fd4377-f606-4d10-93db-f23555a1e1b3",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:08.543890305Z",
      "endTime": "2026-10-16T11:48:09.8244489Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3
            correlation ID: e1749404-9e54-474d-bcf9-84f47251ca01
            client request ID: 8cba2cc4-4919-4b95-82c2-d9427a0e27b0
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt
Unexpected error:
    <*fmt.wrapError | 0x3829cd510c40>: 
    failed to create HCP cluster des-encrypt: failed waiting for cluster="des-encrypt" in resourcegroup="des-encrypt-4gqdqvkfqdht" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3",
      "name": "43fd4377-f606-4d10-93db-f23555a1e1b3",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:08.543890305Z",
      "endTime": "2026-10-16T11:48:09.8244489Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3
            correlation ID: e1749404-9e54-474d-bcf9-84f47251ca01
            client request ID: 8cba2cc4-4919-4b95-82c2-d9427a0e27b0
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-f26jnj" with version 4.21.37 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1c652124c5c0>: 
    failed to create HCP cluster cluster-zstream-4-21-f26jnj: failed waiting for cluster="cluster-zstream-4-21-f26jnj" in resourcegroup="rg-zstream-upgrade-4-21-xq96l5g2nl8g" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff",
      "name": "32f3a7f1-4008-4b7b-bf84-4f1342cb3fff",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.444176059Z",
      "endTime": "2026-10-16T11:47:24.728940607Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff
            correlation ID: 83ad100d-6464-4314-bda4-bc37110e46f7
            client request ID: 3c78c9bb-d24a-481a-85ea-5be29a996155
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-f26jnj" with version 4.21.37 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1c652124c5c0>: 
    failed to create HCP cluster cluster-zstream-4-21-f26jnj: failed waiting for cluster="cluster-zstream-4-21-f26jnj" in resourcegroup="rg-zstream-upgrade-4-21-xq96l5g2nl8g" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff",
      "name": "32f3a7f1-4008-4b7b-bf84-4f1342cb3fff",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:15.444176059Z",
      "endTime": "2026-10-16T11:47:24.728940607Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff
            correlation ID: 83ad100d-6464-4314-bda4-bc37110e46f7
            client request ID: 3c78c9bb-d24a-481a-85ea-5be29a996155
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-5-0-rskl6m" with version 5.0.0-rc.5 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x323cfa71cc80>: 
    failed to create HCP cluster cluster-zstream-5-0-rskl6m: failed waiting for cluster="cluster-zstream-5-0-rskl6m" in resourcegroup="rg-zstream-upgrade-5-0-sr6fb558zbkp" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe",
      "name": "3bec8b37-4b2a-458d-8cd1-1393433480fe",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:26.138664953Z",
      "endTime": "2026-10-16T11:47:34.774754546Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe
            correlation ID: e09bf7d2-5b59-4164-bb62-51bfaa1958d8
            client request ID: dc11d72e-3bf9-4874-b3c8-0dfb9ea5435b
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-5-0-rskl6m" with version 5.0.0-rc.5 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x323cfa71cc80>: 
    failed to create HCP cluster cluster-zstream-5-0-rskl6m: failed waiting for cluster="cluster-zstream-5-0-rskl6m" in resourcegroup="rg-zstream-upgrade-5-0-sr6fb558zbkp" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe",
      "name": "3bec8b37-4b2a-458d-8cd1-1393433480fe",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:26.138664953Z",
      "endTime": "2026-10-16T11:47:34.774754546Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe
            correlation ID: e09bf7d2-5b59-4164-bb62-51bfaa1958d8
            client request ID: dc11d72e-3bf9-4874-b3c8-0dfb9ea5435b
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster
Unexpected error:
    <*fmt.wrapError | 0x3e9cd3298520>: 
    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster="oidc-wi-cluster" in resourcegroup="oidc-wi-dxm54wcws6hs" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb",
      "name": "9314458c-cab4-4fbd-b0e7-c2c9434231fb",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.929706477Z",
      "endTime": "2026-10-16T12:43:31.118752307Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb
            correlation ID: 7a91ca7e-bafb-4c9a-b8db-f6e7ad6f8b52
            client request ID: 39bf8f20-42d5-4454-a015-9f651c57e1cb
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster
Unexpected error:
    <*fmt.wrapError | 0x3e9cd3298520>: 
    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster="oidc-wi-cluster" in resourcegroup="oidc-wi-dxm54wcws6hs" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb",
      "name": "9314458c-cab4-4fbd-b0e7-c2c9434231fb",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.929706477Z",
      "endTime": "2026-10-16T12:43:31.118752307Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb
            correlation ID: 7a91ca7e-bafb-4c9a-b8db-f6e7ad6f8b52
            client request ID: 39bf8f20-42d5-4454-a015-9f651c57e1cb
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster "arm64-vm-hcp-cluster"
Unexpected error:
    <*fmt.wrapError | 0x2f141cf40a40>: 
    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster="arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-s45bf5gctwrz" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba",
      "name": "d5d7fa0e-0178-458e-94e7-2895ae2998ba",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.349858893Z",
      "endTime": "2026-10-16T11:47:34.736973041Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba
            correlation ID: 2d09fb79-4f4d-4e95-9537-b8e2ea39de31
            client request ID: d79af74d-f260-41fd-865c-d23209b51fe0
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster "arm64-vm-hcp-cluster"
Unexpected error:
    <*fmt.wrapError | 0x2f141cf40a40>: 
    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster="arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-s45bf5gctwrz" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba",
      "name": "d5d7fa0e-0178-458e-94e7-2895ae2998ba",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.349858893Z",
      "endTime": "2026-10-16T11:47:34.736973041Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba
            correlation ID: 2d09fb79-4f4d-4e95-9537-b8e2ea39de31
            client request ID: d79af74d-f260-41fd-865c-d23209b51fe0
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-v7kmcc with version 4.21.38
Unexpected error:
    <*fmt.wrapError | 0x210c156bc700>: 
    failed to create HCP cluster np-version-upgrade-cluster-v7kmcc: failed waiting for cluster="np-version-upgrade-cluster-v7kmcc" in resourcegroup="rg-np-version-upgrade-v7kmcc-xsth7mmz6mwq" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27",
      "name": "21480318-8ca1-438b-83d1-c0d4afadcb27",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.86865973Z",
      "endTime": "2026-10-16T12:43:31.120266148Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27
            correlation ID: 88d3443b-33b0-4187-ae32-473011f1f8a9
            client request ID: a93ba081-83e5-408e-b984-1c88d2bcbd35
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-v7kmcc with version 4.21.38
Unexpected error:
    <*fmt.wrapError | 0x210c156bc700>: 
    failed to create HCP cluster np-version-upgrade-cluster-v7kmcc: failed waiting for cluster="np-version-upgrade-cluster-v7kmcc" in resourcegroup="rg-np-version-upgrade-v7kmcc-xsth7mmz6mwq" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27",
      "name": "21480318-8ca1-438b-83d1-c0d4afadcb27",
      "status": "Failed",
      "startTime": "2026-10-16T12:14:29.86865973Z",
      "endTime": "2026-10-16T12:43:31.120266148Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27
            correlation ID: 88d3443b-33b0-4187-ae32-473011f1f8a9
            client request ID: a93ba081-83e5-408e-b984-1c88d2bcbd35
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster "agg-cluster" with aggregated settings
Unexpected error:
    <*fmt.wrapError | 0x16706e68afa0>: 
    failed waiting for cluster="agg-cluster" in resourcegroup="feature-aggregation-6dw2bx5ds9kb" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc",
      "name": "edc5b1cd-acae-47b3-aa88-4662fa83e6fc",
      "status": "Failed",
      "startTime": "2026-10-16T11:21:30.614053227Z",
      "endTime": "2026-10-16T11:50:39.851463582Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc
            correlation ID: d0456136-5d4d-427b-b7a5-1c6f68dd0361
            client request ID: 83b4aaf0-43c3-43b5-a143-55b6ae1d78d1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster "agg-cluster" with aggregated settings
Unexpected error:
    <*fmt.wrapError | 0x16706e68afa0>: 
    failed waiting for cluster="agg-cluster" in resourcegroup="feature-aggregation-6dw2bx5ds9kb" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc",
      "name": "edc5b1cd-acae-47b3-aa88-4662fa83e6fc",
      "status": "Failed",
      "startTime": "2026-10-16T11:21:30.614053227Z",
      "endTime": "2026-10-16T11:50:39.851463582Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc
            correlation ID: d0456136-5d4d-427b-b7a5-1c6f68dd0361
            client request ID: 83b4aaf0-43c3-43b5-a143-55b6ae1d78d1
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x345a0cb74ca0>: 
    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster="cordon-hcp-cluster" in resourcegroup="rg-cordon-test-f5wd6886vcn2" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5",
      "name": "09427242-c235-4b25-9b05-5fc93af25ca5",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:52.120671637Z",
      "endTime": "2026-10-16T11:47:59.844546185Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5
            correlation ID: be95e7ca-864f-4e1e-8ac2-79d1ca2c5a2a
            client request ID: 24b8da34-4220-4d6a-84a8-6aeb131431d9
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x345a0cb74ca0>: 
    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster="cordon-hcp-cluster" in resourcegroup="rg-cordon-test-f5wd6886vcn2" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5",
      "name": "09427242-c235-4b25-9b05-5fc93af25ca5",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:52.120671637Z",
      "endTime": "2026-10-16T11:47:59.844546185Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5
            correlation ID: be95e7ca-864f-4e1e-8ac2-79d1ca2c5a2a
            client request ID: 24b8da34-4220-4d6a-84a8-6aeb131431d9
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test
Unexpected error:
    <*fmt.wrapError | 0x29ceb663a360>: 
    failed to create HCP cluster ea-listjfw7qd: failed waiting for cluster="ea-listjfw7qd" in resourcegroup="ea-list-dgf4pnnb4pjb" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8",
      "name": "4dac0821-0e5e-4879-8b02-6b86c3f9ddb8",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:51.217847726Z",
      "endTime": "2026-10-16T11:47:59.877781908Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8
            correlation ID: 47f07f5a-d8d8-453a-b9aa-aab49eab730a
            client request ID: 41282790-a233-46d8-b966-573e5286f1f4
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test
Unexpected error:
    <*fmt.wrapError | 0x29ceb663a360>: 
    failed to create HCP cluster ea-listjfw7qd: failed waiting for cluster="ea-listjfw7qd" in resourcegroup="ea-list-dgf4pnnb4pjb" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8",
      "name": "4dac0821-0e5e-4879-8b02-6b86c3f9ddb8",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:51.217847726Z",
      "endTime": "2026-10-16T11:47:59.877781908Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8
            correlation ID: 47f07f5a-d8d8-453a-b9aa-aab49eab730a
            client request ID: 41282790-a233-46d8-b966-573e5286f1f4
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-5-1-rf4s5g" at install version 5.0
Unexpected error:
    <*fmt.wrapError | 0x3a0d098322a0>: 
    failed to create HCP cluster cp-ystream-upgrade-5-1-rf4s5g: failed waiting for cluster="cp-ystream-upgrade-5-1-rf4s5g" in resourcegroup="rg-cp-ystream-upgrade-5-1-zfstnv56jv54" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562",
      "name": "5b215067-e4fb-4ba6-a3c8-88327ce04562",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:44.075775976Z",
      "endTime": "2026-10-16T11:47:49.83736828Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562
            correlation ID: 1b44b5b8-3964-4666-b567-9729ea3b6cfb
            client request ID: 420bba0b-936e-4bc8-ab33-d28994da420f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-5-1-rf4s5g" at install version 5.0
Unexpected error:
    <*fmt.wrapError | 0x3a0d098322a0>: 
    failed to create HCP cluster cp-ystream-upgrade-5-1-rf4s5g: failed waiting for cluster="cp-ystream-upgrade-5-1-rf4s5g" in resourcegroup="rg-cp-ystream-upgrade-5-1-zfstnv56jv54" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562",
      "name": "5b215067-e4fb-4ba6-a3c8-88327ce04562",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:44.075775976Z",
      "endTime": "2026-10-16T11:47:49.83736828Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562
            correlation ID: 1b44b5b8-3964-4666-b567-9729ea3b6cfb
            client request ID: 420bba0b-936e-4bc8-ab33-d28994da420f
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test
Unexpected error:
    <*fmt.wrapError | 0xe0368d38100>: 
    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster="pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-h4bzxzqvfdcw" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937",
      "name": "eb0f91c1-3b8b-4469-a615-126ec236e937",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:34.602862036Z",
      "endTime": "2026-10-16T11:47:35.486035854Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937
            correlation ID: 86eb3103-6942-44ec-a43b-2ba190a4ae32
            client request ID: 4d7beedb-0e11-4250-97e6-1589ca4e6668
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test
Unexpected error:
    <*fmt.wrapError | 0xe0368d38100>: 
    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster="pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-h4bzxzqvfdcw" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937",
      "name": "eb0f91c1-3b8b-4469-a615-126ec236e937",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:34.602862036Z",
      "endTime": "2026-10-16T11:47:35.486035854Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937
            correlation ID: 86eb3103-6942-44ec-a43b-2ba190a4ae32
            client request ID: 4d7beedb-0e11-4250-97e6-1589ca4e6668
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test
Unexpected error:
    <*fmt.wrapError | 0x356517fb8480>: 
    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-gg2pb4grctp4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5",
      "name": "cd8f2d5b-465a-406e-9cc2-e6c9587606f5",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:52.22518707Z",
      "endTime": "2026-10-16T11:47:59.816411083Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5
            correlation ID: 9cbbfb0c-2b84-41ef-8052-4fc6765ee876
            client request ID: 78dd1326-407d-4271-9447-caf037d2818a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test
Unexpected error:
    <*fmt.wrapError | 0x356517fb8480>: 
    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-gg2pb4grctp4" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5",
      "name": "cd8f2d5b-465a-406e-9cc2-e6c9587606f5",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:52.22518707Z",
      "endTime": "2026-10-16T11:47:59.816411083Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5
            correlation ID: 9cbbfb0c-2b84-41ef-8052-4fc6765ee876
            client request ID: 78dd1326-407d-4271-9447-caf037d2818a
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster "sre-hcp-cluster-bootdiag" for boot diagnostics test
Unexpected error:
    <*fmt.wrapError | 0x3e5972a223c0>: 
    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster="sre-hcp-cluster-bootdiag" in resourcegroup="admin-api-serialconsole-bootdiag-7j8469sk5tqr" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515",
      "name": "3838bb8e-dfbd-4e5a-9339-671595a03515",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.238054549Z",
      "endTime": "2026-10-16T11:47:34.729902173Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515
            correlation ID: 99bb95a5-8b87-4113-8926-d7de49dc11ce
            client request ID: 758d0f34-5d0d-498d-a7d0-60f7ca767673
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster "sre-hcp-cluster-bootdiag" for boot diagnostics test
Unexpected error:
    <*fmt.wrapError | 0x3e5972a223c0>: 
    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster="sre-hcp-cluster-bootdiag" in resourcegroup="admin-api-serialconsole-bootdiag-7j8469sk5tqr" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515",
      "name": "3838bb8e-dfbd-4e5a-9339-671595a03515",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:25.238054549Z",
      "endTime": "2026-10-16T11:47:34.729902173Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515
            correlation ID: 99bb95a5-8b87-4113-8926-d7de49dc11ce
            client request ID: 758d0f34-5d0d-498d-a7d0-60f7ca767673
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-22-mggmm7" at install version 4.21
Unexpected error:
    <*fmt.wrapError | 0xd7e57968900>: 
    failed to create HCP cluster cp-ystream-upgrade-4-22-mggmm7: failed waiting for cluster="cp-ystream-upgrade-4-22-mggmm7" in resourcegroup="rg-cp-ystream-upgrade-4-22-xjmtnnp8qggs" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572",
      "name": "542436c6-ac27-49e3-a8f4-9fddb7746572",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:33.806164198Z",
      "endTime": "2026-10-16T11:47:34.778492912Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572
            correlation ID: d197fcb4-7896-4d81-aa49-14527696e626
            client request ID: a66a415f-00c6-4bc2-907a-8ee32e11da3e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-22-mggmm7" at install version 4.21
Unexpected error:
    <*fmt.wrapError | 0xd7e57968900>: 
    failed to create HCP cluster cp-ystream-upgrade-4-22-mggmm7: failed waiting for cluster="cp-ystream-upgrade-4-22-mggmm7" in resourcegroup="rg-cp-ystream-upgrade-4-22-xjmtnnp8qggs" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572",
      "name": "542436c6-ac27-49e3-a8f4-9fddb7746572",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:33.806164198Z",
      "endTime": "2026-10-16T11:47:34.778492912Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572
            correlation ID: d197fcb4-7896-4d81-aa49-14527696e626
            client request ID: a66a415f-00c6-4bc2-907a-8ee32e11da3e
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster
Unexpected error:
    <*fmt.wrapError | 0x249a5b185a0>: 
    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster="np-autoscale-cluster" in resourcegroup="np-autoscaling-pkrtk8j7vt4h" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca",
      "name": "3ed5aa3d-aed2-41b9-8f51-718a07aca6ca",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:42.248685179Z",
      "endTime": "2026-10-16T11:47:49.853732372Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca
            correlation ID: fa48c745-f812-4375-9618-451ce1dbb645
            client request ID: 8468ac1c-82d8-4052-8947-0fbd7b155196
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster
Unexpected error:
    <*fmt.wrapError | 0x249a5b185a0>: 
    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster="np-autoscale-cluster" in resourcegroup="np-autoscaling-pkrtk8j7vt4h" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca",
      "name": "3ed5aa3d-aed2-41b9-8f51-718a07aca6ca",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:42.248685179Z",
      "endTime": "2026-10-16T11:47:49.853732372Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca
            correlation ID: fa48c745-f812-4375-9618-451ce1dbb645
            client request ID: 8468ac1c-82d8-4052-8947-0fbd7b155196
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--sjp6pzqgf2j2/cluster-hs-pre-mr5ffk should provision
Unexpected error:
    <*fmt.wrapError | 0x15cd9f62200>: 
    failed to create HCP cluster cluster-hs-pre-mr5ffk: failed waiting for cluster="cluster-hs-pre-mr5ffk" in resourcegroup="rg-hs-pre--sjp6pzqgf2j2" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912",
      "name": "09d7932b-a3ed-490b-ad59-af083f46f912",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:45.646648105Z",
      "endTime": "2026-10-16T11:47:49.816471411Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912
            correlation ID: dad48d0a-2702-4ac2-bb83-5c8e5a1fcf05
            client request ID: 86ec96bd-5ab0-482b-bdd9-e1e77f04cdcf
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--sjp6pzqgf2j2/cluster-hs-pre-mr5ffk should provision
Unexpected error:
    <*fmt.wrapError | 0x15cd9f62200>: 
    failed to create HCP cluster cluster-hs-pre-mr5ffk: failed waiting for cluster="cluster-hs-pre-mr5ffk" in resourcegroup="rg-hs-pre--sjp6pzqgf2j2" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912",
      "name": "09d7932b-a3ed-490b-ad59-af083f46f912",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:45.646648105Z",
      "endTime": "2026-10-16T11:47:49.816471411Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912
            correlation ID: dad48d0a-2702-4ac2-bb83-5c8e5a1fcf05
            client request ID: 86ec96bd-5ab0-482b-bdd9-e1e77f04cdcf
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-qr5nnd with version 4.21.38
Unexpected error:
    <*fmt.wrapError | 0x1f1c2c46ab00>: 
    failed to create HCP cluster np-version-upgrade-cluster-qr5nnd: failed waiting for cluster="np-version-upgrade-cluster-qr5nnd" in resourcegroup="rg-np-version-upgrade-qr5nnd-l8xjnzcdr6f9" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce",
      "name": "31daeda6-8efc-4902-9484-e978fd605dce",
      "status": "Failed",
      "startTime": "2026-10-16T12:12:31.330566092Z",
      "endTime": "2026-10-16T12:41:31.112317819Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce
            correlation ID: cf017195-c27a-482e-a585-164ee741970d
            client request ID: 2b1b472c-af6e-4011-b54c-755047fe28c2
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-qr5nnd with version 4.21.38
Unexpected error:
    <*fmt.wrapError | 0x1f1c2c46ab00>: 
    failed to create HCP cluster np-version-upgrade-cluster-qr5nnd: failed waiting for cluster="np-version-upgrade-cluster-qr5nnd" in resourcegroup="rg-np-version-upgrade-qr5nnd-l8xjnzcdr6f9" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce",
      "name": "31daeda6-8efc-4902-9484-e978fd605dce",
      "status": "Failed",
      "startTime": "2026-10-16T12:12:31.330566092Z",
      "endTime": "2026-10-16T12:41:31.112317819Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce
            correlation ID: cf017195-c27a-482e-a585-164ee741970d
            client request ID: 2b1b472c-af6e-4011-b54c-755047fe28c2
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:96]: failed to create HCP cluster for external auth test
Unexpected error:
    <*fmt.wrapError | 0x3edf0bbb68a0>: 
    failed to create HCP cluster ea-cluster: failed waiting for cluster="ea-cluster" in resourcegroup="external-auth-cluster-z529cprjbb9b" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7",
      "name": "eff59990-4aa9-4e5e-87d9-e72bb79446b7",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:46.389668007Z",
      "endTime": "2026-10-16T11:47:49.83909108Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7
            correlation ID: 8637dbbe-5f4a-484b-9a7f-0752af5bfdf2
            client request ID: 834ff1df-c90f-4eb9-9740-eca7c029338e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:96]: failed to create HCP cluster for external auth test
Unexpected error:
    <*fmt.wrapError | 0x3edf0bbb68a0>: 
    failed to create HCP cluster ea-cluster: failed waiting for cluster="ea-cluster" in resourcegroup="external-auth-cluster-z529cprjbb9b" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7",
      "name": "eff59990-4aa9-4e5e-87d9-e72bb79446b7",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:46.389668007Z",
      "endTime": "2026-10-16T11:47:49.83909108Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7
            correlation ID: 8637dbbe-5f4a-484b-9a7f-0752af5bfdf2
            client request ID: 834ff1df-c90f-4eb9-9740-eca7c029338e
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-21-vxglcs" at install version 4.20
Unexpected error:
    <*fmt.wrapError | 0x3c50ef51c3c0>: 
    failed to create HCP cluster cp-ystream-upgrade-4-21-vxglcs: failed waiting for cluster="cp-ystream-upgrade-4-21-vxglcs" in resourcegroup="rg-cp-ystream-upgrade-4-21-v7bk8rsxngbc" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2",
      "name": "fc0e3d77-531c-4ab5-be8c-bb319d76c5a2",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:43.252078766Z",
      "endTime": "2026-10-16T11:47:49.83582259Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2
            correlation ID: 273c409a-34e8-4c3e-83cc-b70094e63494
            client request ID: c6c50f5f-b0c7-40d0-a975-1c325d678573
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-21-vxglcs" at install version 4.20
Unexpected error:
    <*fmt.wrapError | 0x3c50ef51c3c0>: 
    failed to create HCP cluster cp-ystream-upgrade-4-21-vxglcs: failed waiting for cluster="cp-ystream-upgrade-4-21-vxglcs" in resourcegroup="rg-cp-ystream-upgrade-4-21-v7bk8rsxngbc" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2",
      "name": "fc0e3d77-531c-4ab5-be8c-bb319d76c5a2",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:43.252078766Z",
      "endTime": "2026-10-16T11:47:49.83582259Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2
            correlation ID: 273c409a-34e8-4c3e-83cc-b70094e63494
            client request ID: c6c50f5f-b0c7-40d0-a975-1c325d678573
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x248dbdd6a2d0>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-p9pjc5tph49f" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-p9pjc5tph49f\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x248dbde747e0>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x248dbdd6a2d0>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-p9pjc5tph49f" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-p9pjc5tph49f\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x248dbde747e0>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster "v20261001" via v20261001
Unexpected error:
    <*fmt.wrapError | 0x3a684268b1e0>: 
    failed to create HCP cluster v20261001: failed waiting for cluster="v20261001" in resourcegroup="v20261001-h7n8bhdfxkhz" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff",
      "name": "24d756d8-8406-403a-803e-d72f60212fff",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.969975943Z",
      "endTime": "2026-10-16T11:47:15.076712864Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff
            correlation ID: 830ad185-3324-4619-a20a-fcad371ddf91
            client request ID: 4185ab5a-305d-4830-a594-26d6ec9cbb8f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster "v20261001" via v20261001
Unexpected error:
    <*fmt.wrapError | 0x3a684268b1e0>: 
    failed to create HCP cluster v20261001: failed waiting for cluster="v20261001" in resourcegroup="v20261001-h7n8bhdfxkhz" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff",
      "name": "24d756d8-8406-403a-803e-d72f60212fff",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:14.969975943Z",
      "endTime": "2026-10-16T11:47:15.076712864Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff
            correlation ID: 830ad185-3324-4619-a20a-fcad371ddf91
            client request ID: 4185ab5a-305d-4830-a594-26d6ec9cbb8f
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0x2fb9619ac660>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-mqjl28m5r6sd" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454",
      "name": "9a2fd85c-d6eb-404b-bbec-4b9f90f3d454",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:55.333209171Z",
      "endTime": "2026-10-16T11:48:59.82480605Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454
            correlation ID: f229ed44-de68-4ac8-91bd-519680e0c83b
            client request ID: 6dc6aa5e-bf11-4175-a256-7174e23bec50
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0x2fb9619ac660>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-mqjl28m5r6sd" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454",
      "name": "9a2fd85c-d6eb-404b-bbec-4b9f90f3d454",
      "status": "Failed",
      "startTime": "2026-10-16T11:19:55.333209171Z",
      "endTime": "2026-10-16T11:48:59.82480605Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454
            correlation ID: f229ed44-de68-4ac8-91bd-519680e0c83b
            client request ID: 6dc6aa5e-bf11-4175-a256-7174e23bec50
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster "private-kas" with private KAS
Unexpected error:
    <*fmt.wrapError | 0xde395ae420>: 
    failed to create HCP cluster private-kas: failed waiting for cluster="private-kas" in resourcegroup="private-kas-sc7wtvpvc55m" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507",
      "name": "1512da1f-5680-47e6-aaa6-c5133ea9f507",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:45.126771542Z",
      "endTime": "2026-10-16T11:47:49.832518532Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507
            correlation ID: aec181d7-e9c4-4a80-b7f9-caa74be2736d
            client request ID: 7881578f-b798-428c-aa0c-b9a97e3da350
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster "private-kas" with private KAS
Unexpected error:
    <*fmt.wrapError | 0xde395ae420>: 
    failed to create HCP cluster private-kas: failed waiting for cluster="private-kas" in resourcegroup="private-kas-sc7wtvpvc55m" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507",
      "name": "1512da1f-5680-47e6-aaa6-c5133ea9f507",
      "status": "Failed",
      "startTime": "2026-10-16T11:18:45.126771542Z",
      "endTime": "2026-10-16T11:47:49.832518532Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507
            correlation ID: aec181d7-e9c4-4a80-b7f9-caa74be2736d
            client request ID: 7881578f-b798-428c-aa0c-b9a97e3da350
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster "cilium-cluster" with no CNI and private etcd
Unexpected error:
    <*fmt.wrapError | 0x2ddbb8172580>: 
    failed waiting for cluster="cilium-cluster" in resourcegroup="complex-cilium-kv-gghkqpprl5hx" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5",
      "name": "0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5",
      "status": "Failed",
      "startTime": "2026-10-16T11:20:15.848246532Z",
      "endTime": "2026-10-16T11:49:19.883725425Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5
            correlation ID: 6a9850a7-3bdb-423f-983c-0f81dde37610
            client request ID: 285c03b5-ab82-40d1-bf50-841a421f3e6b
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster "cilium-cluster" with no CNI and private etcd
Unexpected error:
    <*fmt.wrapError | 0x2ddbb8172580>: 
    failed waiting for cluster="cilium-cluster" in resourcegroup="complex-cilium-kv-gghkqpprl5hx" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5",
      "name": "0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5",
      "status": "Failed",
      "startTime": "2026-10-16T11:20:15.848246532Z",
      "endTime": "2026-10-16T11:49:19.883725425Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5
            correlation ID: 6a9850a7-3bdb-423f-983c-0f81dde37610
            client request ID: 285c03b5-ab82-40d1-bf50-841a421f3e6b
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster "basic-hcp-cluster" to complete creation (timeout '30.000000' minutes)
Unexpected error:
    <*exported.ResponseError | 0x3e31c88336b0>: 
    GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802",
      "name": "dd66ee81-312a-43d3-9e03-4a33287de802",
      "status": "Failed",
      "startTime": "2026-10-16T11:20:24.302026234Z",
      "endTime": "2026-10-16T11:49:29.826835151Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802
            correlation ID: eedbd3ff-6629-45c2-8504-b9a36ee198f7
            client request ID: df570f53-90be-4439-a189-428c0f483b8a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster "basic-hcp-cluster" to complete creation (timeout '30.000000' minutes)
Unexpected error:
    <*exported.ResponseError | 0x3e31c88336b0>: 
    GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802",
      "name": "dd66ee81-312a-43d3-9e03-4a33287de802",
      "status": "Failed",
      "startTime": "2026-10-16T11:20:24.302026234Z",
      "endTime": "2026-10-16T11:49:29.826835151Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802
            correlation ID: eedbd3ff-6629-45c2-8504-b9a36ee198f7
            client request ID: df570f53-90be-4439-a189-428c0f483b8a
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T12:43:13Z
  Ended:       2026-10-16T13:05:11Z
  Severity:    Sev3
  Labels:      alertname="userJourneyClusterDeletionErrors6h30m", cluster="ci01-j9203200-svc", component="slo", long_window="6h", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200", region="westus3", severity="3", short_window="30m", slo="cluster-deletion-errors"
  Description: More than 30% of completed cluster delete operations on ci01-j9203200-svc failed or were canceled over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.

alert fired 3 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T11:31:52Z
  Ended:       2026-10-16T11:54:49Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerRetryHotLoop", cluster="ci01-j9203200-svc", component="backend", name="identityroleassignments", region="westus3", severity="warning"
  Description: Backend controller workqueue identityroleassignments has a retry ratio of > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T11:39:48Z
  Ended:       2026-10-16T11:54:49Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerRetryHotLoop", cluster="ci01-j9203200-svc", component="backend", name="dataplaneoidcfederation", region="westus3", severity="warning"
  Description: Backend controller workqueue dataplaneoidcfederation has a retry ratio of > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-16T12:07:49Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerRetryHotLoop", cluster="ci01-j9203200-svc", component="backend", name="ensuremanagedresourcegroup", region="westus3", severity="warning"
  Description: Backend controller workqueue ensuremanagedresourcegroup has a retry ratio of > 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-16T12:19:18Z
  Severity:    Sev3
  Labels:      alertname="userJourneyClusterProvisionErrors6h30m", cluster="ci01-j9203200-svc", component="slo", long_window="6h", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200", region="westus3", severity="3", short_window="30m", slo="cluster-provision-errors"
  Description: More than 30% of completed cluster create (install) operations on ci01-j9203200-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-16T13:12:08Z
  Severity:    Sev3
  Labels:      alertname="userJourneyClusterDeletionErrors1h5m", cluster="ci01-j9203200-svc", component="slo", long_window="1h", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200", region="westus3", severity="3", short_window="5m", slo="cluster-deletion-errors"
  Description: More than 72% of completed cluster delete operations on ci01-j9203200-svc failed or were canceled over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T11:35:33Z
  Ended:       2026-10-16T11:56:31Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j9203200-svc", component="backend", name="dataplaneoidcfederation", region="westus3", severity="warning"
  Description: Backend controller workqueue dataplaneoidcfederation has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T11:36:30Z
  Ended:       2026-10-16T11:56:31Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j9203200-svc", component="backend", name="clusterdegradedaggregator", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterdegradedaggregator has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-16T11:54:17Z
  Severity:    Sev3
  Labels:      alertname="userJourneyClusterProvisionErrors1h5m", cluster="ci01-j9203200-svc", component="slo", long_window="1h", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200", region="westus3", severity="3", short_window="5m", slo="cluster-provision-errors"
  Description: More than 72% of completed cluster create (install) operations on ci01-j9203200-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.

time=2026-10-16T10:37:16.131Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-10-16T10:37:17.576Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-10-16T10:37:18.879Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6554112-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d
time=2026-10-16T10:39:19.372Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.Resources/deployments/182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d/operationStatuses/08584094598472503244\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.Resources/deployments/cluster-ismqifijm7koe\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j6554112-svc\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\",\\r\\n                \\\"message\\\": \\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\"https://ah-ci01-se-j6554112-1.vault.azure.net\\\\\\\"\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T11:18:08Z
  Ended:       2026-10-16T11:34:08Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j5308928-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T11:18:08Z
  Ended:       2026-10-16T11:34:07Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j5308928-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x3d49e0c9b600>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-cqn2ccl8wpmh" to finish creating: GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8",
      "name": "57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8",
      "status": "Failed",
      "startTime": "2026-10-16T11:01:48.232534133Z",
      "endTime": "2026-10-16T11:20:52.453260342Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NodeDoesNotExist: Node does not exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NotReady: * Deleting: Machine deletion in progress, stage: WaitingForInfrastructureDeletion\n* HealthCheckSucceeded: Health check failed: Node failed to report startup in 10m0s\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8
            correlation ID: f387d1ed-05da-402f-8ada-b632f0d02aae
            client request ID: 36e27b8d-f92f-4579-9e74-48bd79aecb10
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x3d49e0c9b600>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-cqn2ccl8wpmh" to finish creating: GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8",
      "name": "57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8",
      "status": "Failed",
      "startTime": "2026-10-16T11:01:48.232534133Z",
      "endTime": "2026-10-16T11:20:52.453260342Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NodeDoesNotExist: Node does not exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NotReady: * Deleting: Machine deletion in progress, stage: WaitingForInfrastructureDeletion\n* HealthCheckSucceeded: Health check failed: Node failed to report startup in 10m0s\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8
            correlation ID: f387d1ed-05da-402f-8ada-b632f0d02aae
            client request ID: 36e27b8d-f92f-4579-9e74-48bd79aecb10

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1c2c57986ce0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T10:45:41.1909764Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1c2c57986cc0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1c2c56cc4500>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1c2c57986ca0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1c2c596cc000>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1c2c57986ce0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T10:45:41.1909764Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1c2c57986cc0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1c2c56cc4500>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1c2c57986ca0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1c2c596cc000>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool "np-1" for private keyvault cluster "private-kv-cluster"
Unexpected error:
    <*fmt.wrapError | 0x33cb56e17020>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-kv-cluster" in resourcegroup="private-keyvault-4xcd2z54mnjl" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b",
      "name": "ed1bfb49-b931-4647-a8ac-58b583c3b50b",
      "status": "Failed",
      "startTime": "2026-10-16T10:16:52.135966436Z",
      "endTime": "2026-10-16T10:35:53.058795763Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-pw27v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4: Failed: virtualmachine failed to create or update. err: failed to get existing resource private-keyvault-4xcd2z54mnjl--managed/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xcd2z54mnjl--managed/providers/Microsoft.Compute/virtualMachines/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b
            correlation ID: ef5c7325-64f6-4ca1-87ca-c2c3e96bf69a
            client request ID: 392329dd-b2a3-4225-a4e8-154f2f223edb
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool "np-1" for private keyvault cluster "private-kv-cluster"
Unexpected error:
    <*fmt.wrapError | 0x33cb56e17020>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-kv-cluster" in resourcegroup="private-keyvault-4xcd2z54mnjl" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b",
      "name": "ed1bfb49-b931-4647-a8ac-58b583c3b50b",
      "status": "Failed",
      "startTime": "2026-10-16T10:16:52.135966436Z",
      "endTime": "2026-10-16T10:35:53.058795763Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-pw27v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4: Failed: virtualmachine failed to create or update. err: failed to get existing resource private-keyvault-4xcd2z54mnjl--managed/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xcd2z54mnjl--managed/providers/Microsoft.Compute/virtualMachines/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b
            correlation ID: ef5c7325-64f6-4ca1-87ca-c2c3e96bf69a
            client request ID: 392329dd-b2a3-4225-a4e8-154f2f223edb
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x1eddd58959c0>: 
    failed to create NodePool autoscale-noaz: failed waiting for nodepool="autoscale-noaz" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-brlqmmsvjbp7" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5",
      "name": "1ec84baa-1f08-44e4-8884-c7d91686d7f5",
      "status": "Failed",
      "startTime": "2026-10-16T10:26:32.519187505Z",
      "endTime": "2026-10-16T10:45:34.937263417Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5
            correlation ID: 6fd1b613-3112-4dbd-9dff-6f44c7554f29
            client request ID: 0ac1b9cb-8173-4c03-9f5c-ab7b1db7762b
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling
Unexpected error:
    <*fmt.wrapError | 0x1eddd58959c0>: 
    failed to create NodePool autoscale-noaz: failed waiting for nodepool="autoscale-noaz" for cluster "np-autoscale-cluster" in resourcegroup="np-autoscaling-brlqmmsvjbp7" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5",
      "name": "1ec84baa-1f08-44e4-8884-c7d91686d7f5",
      "status": "Failed",
      "startTime": "2026-10-16T10:26:32.519187505Z",
      "endTime": "2026-10-16T10:45:34.937263417Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5
            correlation ID: 6fd1b613-3112-4dbd-9dff-6f44c7554f29
            client request ID: 0ac1b9cb-8173-4c03-9f5c-ab7b1db7762b
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool "np-5mfh5m" for cluster "cluster-candidate-4-23-5mfh5m"
Unexpected error:
    <*fmt.wrapError | 0x83c94b64840>: 
    failed to create NodePool np-5mfh5m: failed waiting for nodepool="np-5mfh5m" for cluster "cluster-candidate-4-23-5mfh5m" in resourcegroup="rg-candidate-4-23-t8lk4npslqvb" to finish creating: GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9",
      "name": "4ba6beec-b4a8-4b19-9703-a8d4018563a9",
      "status": "Failed",
      "startTime": "2026-10-16T10:16:52.661682642Z",
      "endTime": "2026-10-16T10:35:56.734294404Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9
            correlation ID: 2de83bb6-2187-443d-9a87-45e45c14da85
            client request ID: 229ecc64-817b-4213-bc11-c707f778c1d6
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool "np-5mfh5m" for cluster "cluster-candidate-4-23-5mfh5m"
Unexpected error:
    <*fmt.wrapError | 0x83c94b64840>: 
    failed to create NodePool np-5mfh5m: failed waiting for nodepool="np-5mfh5m" for cluster "cluster-candidate-4-23-5mfh5m" in resourcegroup="rg-candidate-4-23-t8lk4npslqvb" to finish creating: GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9",
      "name": "4ba6beec-b4a8-4b19-9703-a8d4018563a9",
      "status": "Failed",
      "startTime": "2026-10-16T10:16:52.661682642Z",
      "endTime": "2026-10-16T10:35:56.734294404Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9
            correlation ID: 2de83bb6-2187-443d-9a87-45e45c14da85
            client request ID: 229ecc64-817b-4213-bc11-c707f778c1d6
occurred

time=2026-10-16T09:14:54.862Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description="Step mirror-image\n  Kind: ImageMirror\n  From {<nil> clustersService.image.registry <nil>}:{<nil> clustersService.image.repository <nil>}@{<nil> clustersService.image.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-10-16T09:15:30.118Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\nLogging into target ACR arohcpsvcdev.\nLogin Succeeded\nMirroring image arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d to arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d.\nThe image will still be available under it's original digest sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d in the target registry.\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 1/5). Retrying in 2s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 2/5). Retrying in 4s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 3/5). Retrying in 8s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed (attempt 4/5). Retrying in 16s...\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d --from-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json --to-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json\n exit status 1"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool "np-1" for autoscaling cluster
Unexpected error:
    <*fmt.wrapError | 0x3408412f86c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-nvxdbn5qmgtb" to finish creating: GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16",
      "name": "a18fbb87-8bb7-4a74-8f21-e96af9bfcf16",
      "status": "Failed",
      "startTime": "2026-10-16T08:39:54.399138921Z",
      "endTime": "2026-10-16T08:58:56.325845484Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16
            correlation ID: 956ae344-8430-4fa1-b7b7-d3523f2ec507
            client request ID: 149dc074-a954-4585-b9e7-50faa51cea7a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool "np-1" for autoscaling cluster
Unexpected error:
    <*fmt.wrapError | 0x3408412f86c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-nvxdbn5qmgtb" to finish creating: GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16",
      "name": "a18fbb87-8bb7-4a74-8f21-e96af9bfcf16",
      "status": "Failed",
      "startTime": "2026-10-16T08:39:54.399138921Z",
      "endTime": "2026-10-16T08:58:56.325845484Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16
            correlation ID: 956ae344-8430-4fa1-b7b7-d3523f2ec507
            client request ID: 149dc074-a954-4585-b9e7-50faa51cea7a
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x21a95a4b0000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x21a95b98e500>{
            s: "dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x21a95a4b0000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x21a95b98e500>{
            s: "dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x11ea08e84580>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        err: <*errors.errorString | 0x11ea09d18c20>{
            s: "dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x11ea08e84580>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        err: <*errors.errorString | 0x11ea09d18c20>{
            s: "dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers >= desired replicas > 0; desired=0 current=0 rows=1",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x8a369199c00>: 
    DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x8a36a1383e0>{
            msg: "DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x8a369199c00>: 
    DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x8a36a1383e0>{
            msg: "DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

alert fired 20 time(s) (18 unknown, 2 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T06:22:59Z
  Ended:       2026-10-16T06:43:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s", pod="azure-disk-csi-driver-controller-57b68cc787-snbg5", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s/azure-disk-csi-driver-controller-57b68cc787-snbg5 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T06:22:59Z
  Ended:       2026-10-16T06:43:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="ignition-server-proxy-7bf77dbb94-nn9mf", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-nn9mf has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i", pod="ignition-server-proxy-5ff57d4f88-chbqq", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-chbqq has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s", pod="azure-disk-csi-driver-controller-57b68cc787-mhkrv", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s/azure-disk-csi-driver-controller-57b68cc787-mhkrv has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i", pod="ignition-server-proxy-5ff57d4f88-xdrhg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-xdrhg has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="ignition-server-proxy-7bf77dbb94-qgvpq", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-qgvpq has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:59Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="kube-controller-manager-d9d7765bd-fkcfl", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/kube-controller-manager-d9d7765bd-fkcfl has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="kube-controller-manager-d9d7765bd-4mm6q", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/kube-controller-manager-d9d7765bd-4mm6q has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="ignition-server-proxy-7bf77dbb94-5mxk8", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-5mxk8 has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="ignition-server-proxy-7bf77dbb94-tst84", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-tst84 has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i", pod="ignition-server-proxy-5ff57d4f88-qd9pm", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-qd9pm has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i", pod="ignition-server-proxy-5ff57d4f88-lwxgz", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-lwxgz has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i", pod="ignition-server-proxy-5ff57d4f88-md46c", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-md46c has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="ignition-server-proxy-7bf77dbb94-llgzf", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-llgzf has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:01Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s", pod="azure-disk-csi-driver-controller-794b54859-f2fnv", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s/azure-disk-csi-driver-controller-794b54859-f2fnv has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i", pod="ignition-server-proxy-5ff57d4f88-2clfg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-2clfg has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:40:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i", pod="ignition-server-proxy-5ff57d4f88-sb22k", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-sb22k has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-16T06:23:00Z
  Ended:       2026-10-16T06:43:58Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6330496-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t", pod="kube-controller-manager-d9d7765bd-nw48g", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/kube-controller-manager-d9d7765bd-nw48g has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x202498ad04a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T05:58:01.2930251Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x202498ad0480>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x202497dfe410>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x202498ad0460>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x202499acacc0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x202498ad04a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T05:58:01.2930251Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x202498ad0480>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x202497dfe410>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x202498ad0460>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x202499acacc0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x1583fe428d80>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-qxkjhfl8b89k" to finish creating: GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204",
      "name": "fb248cdb-708c-4e1a-a191-a9d66edb0204",
      "status": "Failed",
      "startTime": "2026-10-16T06:10:02.689570443Z",
      "endTime": "2026-10-16T06:29:04.095727153Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204
            correlation ID: 22bcc657-19c6-431f-9432-afe9f54b1a18
            client request ID: 0f3d0f1f-4c8a-4108-a1df-7f5cda2678a1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x1583fe428d80>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-qxkjhfl8b89k" to finish creating: GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204",
      "name": "fb248cdb-708c-4e1a-a191-a9d66edb0204",
      "status": "Failed",
      "startTime": "2026-10-16T06:10:02.689570443Z",
      "endTime": "2026-10-16T06:29:04.095727153Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204
            correlation ID: 22bcc657-19c6-431f-9432-afe9f54b1a18
            client request ID: 0f3d0f1f-4c8a-4108-a1df-7f5cda2678a1
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T05:28:35Z
  Ended:       2026-10-16T05:43:39Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j8820608-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

time=2026-10-16T04:32:52.168Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T04:32:53.669Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-10-16T04:32:54.816Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3854848%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a
time=2026-10-16T04:48:28.496Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3854848/providers/Microsoft.Resources/deployments/52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a/operationStatuses/08584094817114313637\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3854848/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3854848/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3854848\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Timeout\\\",\\r\\n                \\\"message\\\": \\\"Operation is being Canceled due to timeout.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x34648f7ca8a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T04:59:51.6607748Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x34648f7ca880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x34648e72c7d0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x34648f7ca860>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3464907e0690>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x34648f7ca8a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T04:59:51.6607748Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x34648f7ca880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x34648e72c7d0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x34648f7ca860>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3464907e0690>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

alert fired 10 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-fksqp", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-fksqp has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-x9rnt", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-x9rnt has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-v5kgz", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-v5kgz has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-tmxw7", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-tmxw7 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-2s6gb", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-2s6gb has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-5gsvd", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-5gsvd has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-wh5lq", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-wh5lq has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-5ljrz", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-5ljrz has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-jncwh", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-jncwh has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-16T05:20:25Z
  Ended:       2026-10-16T05:44:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7175168-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z", pod="ignition-server-557b99d5f7-x2w2x", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-x2w2x has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x16b0282ea00>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T04:23:19.3198868Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x16b0282e9e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x16b028e0910>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x16b0282e9c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x16b04458db0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x16b0282ea00>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-16T04:23:19.3198868Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x16b0282e9e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x16b028e0910>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x16b0282e9c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x16b04458db0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

time=2026-10-16T03:17:16.536Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T03:17:18.501Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-10-16T03:17:19.523Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6526208%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57
time=2026-10-16T03:35:23.849Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6526208/providers/Microsoft.Resources/deployments/23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57/operationStatuses/08584094862465785031\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6526208/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6526208/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6526208\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Timeout\\\",\\r\\n                \\\"message\\\": \\\"Operation is being Canceled due to timeout.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-10-16T02:17:33.774Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T02:17:35.249Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-10-16T02:17:36.184Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5876608%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8
time=2026-10-16T02:33:38.292Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5876608/providers/Microsoft.Resources/deployments/11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8/operationStatuses/08584094898298983239\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5876608/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5876608/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5876608\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Timeout\\\",\\r\\n                \\\"message\\\": \\\"Operation is being Canceled due to timeout.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-10-16T02:12:04.133Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T02:12:05.706Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-10-16T02:12:06.465Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fd4c8415653ce521dcac897b418e014c2ef331c8977fc03faf9a6142657d4dcd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1778560%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffd4c8415653ce521dcac897b418e014c2ef331c8977fc03faf9a6142657d4dcd
time=2026-10-16T02:29:09.429Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1778560/providers/Microsoft.Resources/deployments/fd4c8415653ce521dcac897b418e014c2ef331c8977fc03faf9a6142657d4dcd/operationStatuses/08584094901594831073\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1778560/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1778560/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1778560\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Timeout\\\",\\r\\n                \\\"message\\\": \\\"Operation is being Canceled due to timeout.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-10-16T02:15:07.162Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-10-16T02:15:08.937Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-10-16T02:15:09.609Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d5db28dfe0cc3780f80baae497ef5a933a88e4b7ad9d7697cd455d31b3b34457 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7378816%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd5db28dfe0cc3780f80baae497ef5a933a88e4b7ad9d7697cd455d31b3b34457
time=2026-10-16T02:30:42.385Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7378816/providers/Microsoft.Resources/deployments/d5db28dfe0cc3780f80baae497ef5a933a88e4b7ad9d7697cd455d31b3b34457/operationStatuses/08584094899762652758\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7378816/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7378816/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7378816\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Timeout\\\",\\r\\n                \\\"message\\\": \\\"Operation is being Canceled due to timeout.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

alert fired 11 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T02:15:00Z
  Ended:       2026-10-16T02:21:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g", pod="ignition-server-proxy-57c48555fb-9c282", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-9c282 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T02:15:00Z
  Ended:       2026-10-16T02:21:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g", pod="kube-controller-manager-54f5967dd5-6m76d", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/kube-controller-manager-54f5967dd5-6m76d has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-16T02:15:00Z
  Ended:       2026-10-16T02:21:02Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g", pod="ignition-server-proxy-57c48555fb-skdc9", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-skdc9 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-16T02:15:00Z
  Ended:       2026-10-16T02:28:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q", pod="azure-disk-csi-driver-operator-799c47857f-c7ffd", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/azure-disk-csi-driver-operator-799c47857f-c7ffd has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-16T02:15:00Z
  Ended:       2026-10-16T02:21:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g", pod="kube-controller-manager-54f5967dd5-f9767", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/kube-controller-manager-54f5967dd5-f9767 has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-16T02:15:00Z
  Ended:       2026-10-16T02:21:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g", pod="kube-controller-manager-54f5967dd5-58bpn", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/kube-controller-manager-54f5967dd5-58bpn has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-16T02:15:01Z
  Ended:       2026-10-16T02:28:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q", pod="csi-snapshot-controller-645c7d8759-992mc", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/csi-snapshot-controller-645c7d8759-992mc has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-16T02:15:01Z
  Ended:       2026-10-16T02:21:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g", pod="ignition-server-proxy-57c48555fb-vr7j9", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-vr7j9 has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-16T02:15:01Z
  Ended:       2026-10-16T02:21:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g", pod="ignition-server-proxy-57c48555fb-2flbm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-2flbm has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-16T02:15:01Z
  Ended:       2026-10-16T02:28:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q", pod="csi-snapshot-controller-645c7d8759-lwl76", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/csi-snapshot-controller-645c7d8759-lwl76 has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-16T02:15:01Z
  Ended:       2026-10-16T02:28:00Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2423808-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q", pod="azure-file-csi-driver-operator-75d8977b68-cmb5g", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/azure-file-csi-driver-operator-75d8977b68-cmb5g has been in a non-ready state for longer than 5 minutes.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T02:38:55Z
  Ended:       2026-10-16T03:16:55Z
  Severity:    Sev3
  Labels:      alertname="HCPClusterOperatorUnavailable", cluster="ci01-j2423808-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj30j2k6sdjh9r7vsbj6npes2m40m2-p2r6x1t9q6d6f9l", region="centralus", severity="warning"
  Description: 1 cluster operator(s) on hosted cluster ocm-arohcpci01-2tfj30j2k6sdjh9r7vsbj6npes2m40m2-p2r6x1t9q6d6f9l (management cluster ci01-j2423808-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T02:40:29Z
  Ended:       2026-10-16T03:15:31Z
  Severity:    Sev3
  Labels:      alertname="HCPClusterOperatorUnavailable", cluster="ci01-j3528064-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3qrd1lfn90p1i58kr795ndsi2u3g-f4z5z8b0a1u0o2v", region="westus3", severity="warning"
  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tfj3qrd1lfn90p1i58kr795ndsi2u3g-f4z5z8b0a1u0o2v (management cluster ci01-j3528064-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.

alert fired 82 time(s) (64 unknown, 18 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T02:10:05Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-m9lxt", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-m9lxt has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T02:10:05Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-468dp", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-468dp has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-16T02:10:05Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-mcp7n", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-mcp7n has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-2lx2c", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-2lx2c has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-nw6s6", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-nw6s6 has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-7bc8c", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-7bc8c has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:04Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="network-node-identity-5b8749cb94-wvtfh", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-wvtfh has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-55wn4", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-55wn4 has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-n2fw5", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-n2fw5 has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-45l5z", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-45l5z has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-5k68x", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5k68x has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-59vdg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-59vdg has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-d54pm", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-d54pm has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-xqpw9", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-xqpw9 has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-k2h6l", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-k2h6l has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-mntnm", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-mntnm has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-5bbbw", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5bbbw has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-bjfl2", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bjfl2 has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="network-node-identity-5b8749cb94-xmxx2", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-xmxx2 has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-dfxm6", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-dfxm6 has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-nwmdr", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-nwmdr has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-ddvnx", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ddvnx has been in a non-ready state for longer than 5 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-mnjm6", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-mnjm6 has been in a non-ready state for longer than 5 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-tkqrb", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-tkqrb has been in a non-ready state for longer than 5 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-blbx4", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-blbx4 has been in a non-ready state for longer than 5 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-gzwn6", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-gzwn6 has been in a non-ready state for longer than 5 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-l9ch7", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-l9ch7 has been in a non-ready state for longer than 5 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-nq9pw", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-nq9pw has been in a non-ready state for longer than 5 minutes.
Firing 29:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-xtf8z", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-xtf8z has been in a non-ready state for longer than 5 minutes.
Firing 30:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-gm47p", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-gm47p has been in a non-ready state for longer than 5 minutes.
Firing 31:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-bclsd", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bclsd has been in a non-ready state for longer than 5 minutes.
Firing 32:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-ht9sj", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ht9sj has been in a non-ready state for longer than 5 minutes.
Firing 33:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-p4gcc", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-p4gcc has been in a non-ready state for longer than 5 minutes.
Firing 34:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-qr5vd", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-qr5vd has been in a non-ready state for longer than 5 minutes.
Firing 35:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-jncsq", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-jncsq has been in a non-ready state for longer than 5 minutes.
Firing 36:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-46plt", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-46plt has been in a non-ready state for longer than 5 minutes.
Firing 37:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-l2zcs", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-l2zcs has been in a non-ready state for longer than 5 minutes.
Firing 38:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-l4r92", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-l4r92 has been in a non-ready state for longer than 5 minutes.
Firing 39:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="network-node-identity-5b8749cb94-v996p", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-v996p has been in a non-ready state for longer than 5 minutes.
Firing 40:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="network-node-identity-5b8749cb94-czbkf", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-czbkf has been in a non-ready state for longer than 5 minutes.
Firing 41:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-dj84f", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-dj84f has been in a non-ready state for longer than 5 minutes.
Firing 42:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-p8zp2", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-p8zp2 has been in a non-ready state for longer than 5 minutes.
Firing 43:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-lmxtt", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-lmxtt has been in a non-ready state for longer than 5 minutes.
Firing 44:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-q7sq8", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-q7sq8 has been in a non-ready state for longer than 5 minutes.
Firing 45:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-bt4kv", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bt4kv has been in a non-ready state for longer than 5 minutes.
Firing 46:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-ztqwt", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ztqwt has been in a non-ready state for longer than 5 minutes.
Firing 47:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="network-node-identity-5b8749cb94-vtzrd", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-vtzrd has been in a non-ready state for longer than 5 minutes.
Firing 48:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-hsncg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-hsncg has been in a non-ready state for longer than 5 minutes.
Firing 49:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-bzqvx", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bzqvx has been in a non-ready state for longer than 5 minutes.
Firing 50:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-sg28h", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-sg28h has been in a non-ready state for longer than 5 minutes.
Firing 51:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="network-node-identity-5b8749cb94-bq57x", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-bq57x has been in a non-ready state for longer than 5 minutes.
Firing 52:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-zr8bf", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-zr8bf has been in a non-ready state for longer than 5 minutes.
Firing 53:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-sbsz2", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-sbsz2 has been in a non-ready state for longer than 5 minutes.
Firing 54:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-5w62k", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5w62k has been in a non-ready state for longer than 5 minutes.
Firing 55:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-ssb27", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ssb27 has been in a non-ready state for longer than 5 minutes.
Firing 56:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-2lnh7", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-2lnh7 has been in a non-ready state for longer than 5 minutes.
Firing 57:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-9wkkg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-9wkkg has been in a non-ready state for longer than 5 minutes.
Firing 58:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-flxcj", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-flxcj has been in a non-ready state for longer than 5 minutes.
Firing 59:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-5dvp5", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5dvp5 has been in a non-ready state for longer than 5 minutes.
Firing 60:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-6gc2q", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-6gc2q has been in a non-ready state for longer than 5 minutes.
Firing 61:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-s9ccj", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-s9ccj has been in a non-ready state for longer than 5 minutes.
Firing 62:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-5t86z", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5t86z has been in a non-ready state for longer than 5 minutes.
Firing 63:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="ovnkube-control-plane-6685c9694f-s6t9p", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-s6t9p has been in a non-ready state for longer than 5 minutes.
Firing 64:
  State:       Resolved
  Started:     2026-10-16T02:10:06Z
  Ended:       2026-10-16T02:34:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3528064-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e", pod="network-node-identity-5b8749cb94-2v49b", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-2v49b has been in a non-ready state for longer than 5 minutes.

alert fired 9 time(s) (3 unknown, 6 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T02:01:55Z
  Ended:       2026-10-16T02:17:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6640000-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster", pod="ignition-server-proxy-56c8c65f87-54tt4", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster/ignition-server-proxy-56c8c65f87-54tt4 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T02:01:55Z
  Ended:       2026-10-16T02:17:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6640000-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster", pod="ignition-server-proxy-56c8c65f87-nhp86", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster/ignition-server-proxy-56c8c65f87-nhp86 has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-16T02:01:55Z
  Ended:       2026-10-16T02:17:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6640000-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster", pod="ignition-server-proxy-56c8c65f87-27bzh", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster/ignition-server-proxy-56c8c65f87-27bzh has been in a non-ready state for longer than 5 minutes.

alert fired 54 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-8prks", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-8prks has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-nb952", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-nb952 has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-wb8jp", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-wb8jp has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-b8j7b", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-b8j7b has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-tdtms", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-tdtms has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-b6c6j", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-b6c6j has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-2pn5b", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-2pn5b has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-t5wcc", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-t5wcc has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-vdvhv", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-vdvhv has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-rlpxn", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-rlpxn has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-bvbtx", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-bvbtx has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k", pod="network-node-identity-6cc47c64d4-7pj68", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-7pj68 has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k", pod="network-node-identity-6cc47c64d4-rxvck", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-rxvck has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k", pod="network-node-identity-6cc47c64d4-x7gdj", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-x7gdj has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-cbqv5", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-cbqv5 has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-l8frm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-l8frm has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-cnrt7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-cnrt7 has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-fzpnb", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-fzpnb has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k", pod="network-node-identity-6cc47c64d4-z5fsz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-z5fsz has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-qgw4k", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-qgw4k has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:12:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-6zh7n", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-6zh7n has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-ngdpw", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-ngdpw has been in a non-ready state for longer than 5 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-llxb8", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-llxb8 has been in a non-ready state for longer than 5 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-16T01:55:18Z
  Ended:       2026-10-16T02:09:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n", pod="network-node-identity-6c9c9b694-c7lsz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-c7lsz has been in a non-ready state for longer than 5 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-49gpg", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-49gpg has been in a non-ready state for longer than 5 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-jbs6b", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-jbs6b has been in a non-ready state for longer than 5 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-nvqxc", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-nvqxc has been in a non-ready state for longer than 5 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-7s5p2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-7s5p2 has been in a non-ready state for longer than 5 minutes.
Firing 29:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-f5k6n", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-f5k6n has been in a non-ready state for longer than 5 minutes.
Firing 30:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-pg6x2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-pg6x2 has been in a non-ready state for longer than 5 minutes.
Firing 31:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-jgx64", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-jgx64 has been in a non-ready state for longer than 5 minutes.
Firing 32:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-fm5rr", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-fm5rr has been in a non-ready state for longer than 5 minutes.
Firing 33:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-xb9s2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-xb9s2 has been in a non-ready state for longer than 5 minutes.
Firing 34:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-c7r7c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-c7r7c has been in a non-ready state for longer than 5 minutes.
Firing 35:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-qch8q", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-qch8q has been in a non-ready state for longer than 5 minutes.
Firing 36:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-9bsh7", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9bsh7 has been in a non-ready state for longer than 5 minutes.
Firing 37:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-xmx8g", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-xmx8g has been in a non-ready state for longer than 5 minutes.
Firing 38:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-9c99d", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9c99d has been in a non-ready state for longer than 5 minutes.
Firing 39:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-tht7j", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-tht7j has been in a non-ready state for longer than 5 minutes.
Firing 40:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-9rc8c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9rc8c has been in a non-ready state for longer than 5 minutes.
Firing 41:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-bsw8c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-bsw8c has been in a non-ready state for longer than 5 minutes.
Firing 42:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-7dr6c", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-7dr6c has been in a non-ready state for longer than 5 minutes.
Firing 43:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-8kmpk", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8kmpk has been in a non-ready state for longer than 5 minutes.
Firing 44:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-r8vf2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-r8vf2 has been in a non-ready state for longer than 5 minutes.
Firing 45:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-2pf6l", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-2pf6l has been in a non-ready state for longer than 5 minutes.
Firing 46:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-9zbkd", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9zbkd has been in a non-ready state for longer than 5 minutes.
Firing 47:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-zjwxx", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-zjwxx has been in a non-ready state for longer than 5 minutes.
Firing 48:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-xdz7r", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-xdz7r has been in a non-ready state for longer than 5 minutes.
Firing 49:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-wg7qs", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-wg7qs has been in a non-ready state for longer than 5 minutes.
Firing 50:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-8zmzm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8zmzm has been in a non-ready state for longer than 5 minutes.
Firing 51:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-68vhm", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-68vhm has been in a non-ready state for longer than 5 minutes.
Firing 52:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-8p9wj", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8p9wj has been in a non-ready state for longer than 5 minutes.
Firing 53:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-8dqjv", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8dqjv has been in a non-ready state for longer than 5 minutes.
Firing 54:
  State:       Resolved
  Started:     2026-10-16T01:56:18Z
  Ended:       2026-10-16T02:19:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8280832-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e", pod="network-node-identity-56cf76dccd-84lvn", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-84lvn has been in a non-ready state for longer than 5 minutes.