fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:
    <*errors.joinError | 0x2b12a0180120>: 
    resource group "rg-candidate-4-22-9q8kddc4rfc6": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-22-hscqkp" in resourcegroup="rg-candidate-4-22-9q8kddc4rfc6" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1",
      "name": "46e9ad9e-9f53-4038-ba36-f40dec0ff2b1",
      "status": "Failed",
      "startTime": "2026-10-28T17:51:06.474152018Z",
      "endTime": "2026-10-28T18:15:11.708953381Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1
            correlation ID: 4741ead8-d72b-47f6-84e5-df7418434c6d
            client request ID: 9a224eab-3688-4c8c-9f47-6f8238e2c607
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:
    <*errors.joinError | 0x2b12a0180120>: 
    resource group "rg-candidate-4-22-9q8kddc4rfc6": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-22-hscqkp" in resourcegroup="rg-candidate-4-22-9q8kddc4rfc6" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1",
      "name": "46e9ad9e-9f53-4038-ba36-f40dec0ff2b1",
      "status": "Failed",
      "startTime": "2026-10-28T17:51:06.474152018Z",
      "endTime": "2026-10-28T18:15:11.708953381Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46e9ad9e-9f53-4038-ba36-f40dec0ff2b1
            correlation ID: 4741ead8-d72b-47f6-84e5-df7418434c6d
            client request ID: 9a224eab-3688-4c8c-9f47-6f8238e2c607
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:
    <*errors.joinError | 0x2a43d7c880a8>: 
    resource group "private-kas-2t2bq848xclk": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="private-kas" in resourcegroup="private-kas-2t2bq848xclk" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed",
      "name": "827c464e-64e1-4e42-980f-59af842b0bed",
      "status": "Failed",
      "startTime": "2026-10-28T17:45:46.805727964Z",
      "endTime": "2026-10-28T18:09:51.562148289Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed
            correlation ID: c4e40049-9c6b-42d0-ae1f-bfdcd35c5399
            client request ID: 5bcb7ef3-7dfb-48a5-a7c7-360bec18ff93
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:
    <*errors.joinError | 0x2a43d7c880a8>: 
    resource group "private-kas-2t2bq848xclk": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="private-kas" in resourcegroup="private-kas-2t2bq848xclk" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed",
      "name": "827c464e-64e1-4e42-980f-59af842b0bed",
      "status": "Failed",
      "startTime": "2026-10-28T17:45:46.805727964Z",
      "endTime": "2026-10-28T18:09:51.562148289Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/827c464e-64e1-4e42-980f-59af842b0bed
            correlation ID: c4e40049-9c6b-42d0-ae1f-bfdcd35c5399
            client request ID: 5bcb7ef3-7dfb-48a5-a7c7-360bec18ff93
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:
    <*errors.joinError | 0x17fd8fc22138>: 
    resource group "kusto-logs-2tq8m2fllmvs": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-2tq8m2fllmvs" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1",
      "name": "e027677d-2084-4178-b70c-b4e22dfc37b1",
      "status": "Failed",
      "startTime": "2026-10-28T17:46:42.22634078Z",
      "endTime": "2026-10-28T18:10:50.94970388Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1
            correlation ID: fcf84780-ffda-49de-a03f-17dbeb0ef756
            client request ID: 070116fb-514a-47f7-ae2b-d06066429ea7
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:321]: Unexpected error:
    <*errors.joinError | 0x17fd8fc22138>: 
    resource group "kusto-logs-2tq8m2fllmvs": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-2tq8m2fllmvs" to finish deleting: GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1",
      "name": "e027677d-2084-4178-b70c-b4e22dfc37b1",
      "status": "Failed",
      "startTime": "2026-10-28T17:46:42.22634078Z",
      "endTime": "2026-10-28T18:10:50.94970388Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7963648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e027677d-2084-4178-b70c-b4e22dfc37b1
            correlation ID: fcf84780-ffda-49de-a03f-17dbeb0ef756
            client request ID: 070116fb-514a-47f7-ae2b-d06066429ea7
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls
Unexpected error:
    <*fmt.wrapError | 0x3f06fbed83a0>: 
    failed to create NodePool swift-workers: failed waiting for nodepool="swift-workers" for cluster "engineering-hcp-cluster" in resourcegroup="kusto-logs-4lmptk5vpx7s" to finish creating: GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea",
      "name": "f4b98636-6475-41ec-a151-007f2ef479ea",
      "status": "Failed",
      "startTime": "2026-10-28T19:44:01.234316083Z",
      "endTime": "2026-10-28T20:03:07.974415443Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/virtualMachines/l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: Failed: availabilitysets failed to create or update. err: failed to get existing resource kusto-logs-4lmptk5vpx7s--managed/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/availabilitySets/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1868e77e-aae0-4284-940e-e4cd46ae5baa' exceeded. Please try ag... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea
            correlation ID: 1f8c9a64-07bf-44f3-bca5-a4308cdb0f56
            client request ID: 49aeda1c-19cc-4647-af7d-1d78f910596d
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls
Unexpected error:
    <*fmt.wrapError | 0x3f06fbed83a0>: 
    failed to create NodePool swift-workers: failed waiting for nodepool="swift-workers" for cluster "engineering-hcp-cluster" in resourcegroup="kusto-logs-4lmptk5vpx7s" to finish creating: GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea",
      "name": "f4b98636-6475-41ec-a151-007f2ef479ea",
      "status": "Failed",
      "startTime": "2026-10-28T19:44:01.234316083Z",
      "endTime": "2026-10-28T20:03:07.974415443Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/virtualMachines/l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine l3j9w2r9y4j1z4u-swift-workers-4vqnr-xznbb: Failed: availabilitysets failed to create or update. err: failed to get existing resource kusto-logs-4lmptk5vpx7s--managed/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4lmptk5vpx7s--managed/providers/Microsoft.Compute/availabilitySets/2tnvvns3jn8rgfrctgfo79mkpouiqrqc_l3j9w2r9y4j1z4u-swift-workers-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '1868e77e-aae0-4284-940e-e4cd46ae5baa' exceeded. Please try ag... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7626240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4b98636-6475-41ec-a151-007f2ef479ea
            correlation ID: 1f8c9a64-07bf-44f3-bca5-a4308cdb0f56
            client request ID: 49aeda1c-19cc-4647-af7d-1d78f910596d
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool "np-8844z2" for cluster "cluster-candidate-4-22-8844z2"
Unexpected error:
    <*fmt.wrapError | 0x34a8b7c1a9e0>: 
    failed to create NodePool np-8844z2: failed waiting for nodepool="np-8844z2" for cluster "cluster-candidate-4-22-8844z2" in resourcegroup="rg-candidate-4-22-p8n8vcw2knj6" to finish creating: GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c",
      "name": "e163bbab-c728-4e64-85d2-39934825ad2c",
      "status": "Failed",
      "startTime": "2026-10-28T17:07:17.250607245Z",
      "endTime": "2026-10-28T17:26:25.565644744Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Compute/virtualMachines/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Network/networkInterfaces/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' s... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c
            correlation ID: 8a8572c9-5da8-4651-8233-b7c79a67a3b3
            client request ID: fc96c7e7-a6f5-442a-9e18-6d3eec0c11ee
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool "np-8844z2" for cluster "cluster-candidate-4-22-8844z2"
Unexpected error:
    <*fmt.wrapError | 0x34a8b7c1a9e0>: 
    failed to create NodePool np-8844z2: failed waiting for nodepool="np-8844z2" for cluster "cluster-candidate-4-22-8844z2" in resourcegroup="rg-candidate-4-22-p8n8vcw2knj6" to finish creating: GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c",
      "name": "e163bbab-c728-4e64-85d2-39934825ad2c",
      "status": "Failed",
      "startTime": "2026-10-28T17:07:17.250607245Z",
      "endTime": "2026-10-28T17:26:25.565644744Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Compute/virtualMachines/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-zpqqj to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-p8n8vcw2knj6-candidate-8844z2--managed/providers/Microsoft.Network/networkInterfaces/y8h5w3s1s0z4u9d-np-8844z2-q5kw2-qsnbp-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' s... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7668608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e163bbab-c728-4e64-85d2-39934825ad2c
            correlation ID: 8a8572c9-5da8-4651-8233-b7c79a67a3b3
            client request ID: fc96c7e7-a6f5-442a-9e18-6d3eec0c11ee
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool "nodepool-128GiB" with 128GiB osDisk
Unexpected error:
    <*fmt.wrapError | 0x4218f58d8c0>: 
    failed to create NodePool nodepool-128GiB: failed waiting for nodepool="nodepool-128GiB" for cluster "hcp-cluster-np-128" in resourcegroup="clusternp128-8q6szjwkq6rk" to finish creating: GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99",
      "name": "78051d0a-b2ad-4647-99d4-120efa3f6a99",
      "status": "Failed",
      "startTime": "2026-10-28T21:13:11.691142567Z",
      "endTime": "2026-10-28T21:32:17.541374372Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2: Failed: interfaces failed to create or update. err: failed to get existing resource clusternp128-8q6szjwkq6rk--managed/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8q6szjwkq6rk--managed/providers/Microsoft.Network/networkInterfaces/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3996e871-9d8c-4863-9b56-cf4685281cdc' exceeded. Please try again after '10' seconds after additional t... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99
            correlation ID: 7ef5aa4f-453a-4950-9b06-46a903ca993d
            client request ID: a9dcefcf-ba4d-4111-9c80-6dbba9abd781
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool "nodepool-128GiB" with 128GiB osDisk
Unexpected error:
    <*fmt.wrapError | 0x4218f58d8c0>: 
    failed to create NodePool nodepool-128GiB: failed waiting for nodepool="nodepool-128GiB" for cluster "hcp-cluster-np-128" in resourcegroup="clusternp128-8q6szjwkq6rk" to finish creating: GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99",
      "name": "78051d0a-b2ad-4647-99d4-120efa3f6a99",
      "status": "Failed",
      "startTime": "2026-10-28T21:13:11.691142567Z",
      "endTime": "2026-10-28T21:32:17.541374372Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2: Failed: interfaces failed to create or update. err: failed to get existing resource clusternp128-8q6szjwkq6rk--managed/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8q6szjwkq6rk--managed/providers/Microsoft.Network/networkInterfaces/a7p5v6a9s2f0e3j-nodepool-128gib-brx87-m46c2-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3996e871-9d8c-4863-9b56-cf4685281cdc' exceeded. Please try again after '10' seconds after additional t... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0183936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/78051d0a-b2ad-4647-99d4-120efa3f6a99
            correlation ID: 7ef5aa4f-453a-4950-9b06-46a903ca993d
            client request ID: a9dcefcf-ba4d-4111-9c80-6dbba9abd781
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x26678ea60858>: 
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-f8mftsvbkmbp" to finish creating: GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f",
      "name": "95ed2027-596a-4016-be69-40cc2e92cf3f",
      "status": "Failed",
      "startTime": "2026-10-27T01:20:09.676863434Z",
      "endTime": "2026-10-27T01:39:10.703576848Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f
            correlation ID: 7f67a06a-5085-4c8e-9661-1df8e8a79a0a
            client request ID: 9930c3d6-bc48-4ffb-b0a4-b32c7c60a790
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x26678ea60858>: 
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-f8mftsvbkmbp" to finish creating: GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f",
      "name": "95ed2027-596a-4016-be69-40cc2e92cf3f",
      "status": "Failed",
      "startTime": "2026-10-27T01:20:09.676863434Z",
      "endTime": "2026-10-27T01:39:10.703576848Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine cilium-cluster-cilium-np-j8lrv-nn5dn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-f8mftsvbkmbp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-j8lrv-nn5dn to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2959360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/95ed2027-596a-4016-be69-40cc2e92cf3f
            correlation ID: 7f67a06a-5085-4c8e-9661-1df8e8a79a0a
            client request ID: 9930c3d6-bc48-4ffb-b0a4-b32c7c60a790
occurred

time=2026-10-27T07:31:38.140Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description="Step cluster\n  Kind: ARM\n  Template: templates/mgmt-cluster.bicep\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam"
time=2026-10-27T07:31:38.545Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1
time=2026-10-27T07:31:39.254Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1948032-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0
time=2026-10-27T07:33:09.610Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err="stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1948032-mgmt-1/providers/Microsoft.Resources/deployments/95d11f391192e4ffeb516d68e892ddaaf99516942ada8720f17881ddeb56eda0/operationStatuses/08584085205866172617\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1948032-mgmt-1/providers/Microsoft.Resources/deployments/eventGridPrivateEndpoint\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidRequest\\\",\\r\\n            \\\"message\\\": \\\"Call to Microsoft.EventGrid/namespaces failed. Error message: Required property 'linkIdentifier' not found in JSON. Path 'remotePrivateEndpoint.connectionDetails[0]', line 1, position 932.\\\",\\r\\n            \\\"details\\\": []\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-jz7tkx"
Unexpected error:
    <*errors.joinError | 0x3490511d0108>: 
    VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) timed out after 11m45.097s: DNS A record "*.e2e-dns-4sk7fq" not yet present in zone rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-4sk7fq' does not exist in resource group 'rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq
            correlation ID: 7140a367-bc3d-43f2-b136-ffe5167138f0
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-jz7tkx"
Unexpected error:
    <*errors.joinError | 0x3490511d0108>: 
    VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud) timed out after 11m45.097s: DNS A record "*.e2e-dns-4sk7fq" not yet present in zone rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-4sk7fq' does not exist in resource group 'rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-5sw7d6kw8f6m-candidate-jz7tkx--managed/providers/Microsoft.Network/dnsZones/aro.v8z3q8n3u9i1v9l.eahs.j1816448.hcp.osadev.cloud/A/*.e2e-dns-4sk7fq
            correlation ID: 7140a367-bc3d-43f2-b136-ffe5167138f0
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-7tbfp9"
Unexpected error:
    <*errors.joinError | 0x28672532e420>: 
    VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record "*.e2e-dns-bkpxxt" not yet present in zone rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-bkpxxt' does not exist in resource group 'rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt
            correlation ID: 00f470eb-29ca-4c55-b41f-faa8336d52a1
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-7tbfp9"
Unexpected error:
    <*errors.joinError | 0x28672532e420>: 
    VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record "*.e2e-dns-bkpxxt" not yet present in zone rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-bkpxxt' does not exist in resource group 'rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cwv9mngrfv9r-candidate-7tbfp9--managed/providers/Microsoft.Network/dnsZones/aro.r8a3f2x2r3m7g7z.8i1p.j8906368.hcp.osadev.cloud/A/*.e2e-dns-bkpxxt
            correlation ID: 00f470eb-29ca-4c55-b41f-faa8336d52a1
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-kzwwft"
Unexpected error:
    <*errors.joinError | 0x2fea822a5d0>: 
    VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) timed out after 10m0s: DNS A record "*.e2e-dns-czbqgt" not yet present in zone rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-czbqgt' does not exist in resource group 'rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt
            correlation ID: ef507981-8069-4371-b52a-e18fb184cf37
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-kzwwft"
Unexpected error:
    <*errors.joinError | 0x2fea822a5d0>: 
    VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud) timed out after 10m0s: DNS A record "*.e2e-dns-czbqgt" not yet present in zone rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-czbqgt' does not exist in resource group 'rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-qkhkjtrdhq8n-candidate-kzwwft--managed/providers/Microsoft.Network/dnsZones/aro.m3u7u2n3l0l3p3b.0iw8.j3991936.hcp.osadev.cloud/A/*.e2e-dns-czbqgt
            correlation ID: ef507981-8069-4371-b52a-e18fb184cf37
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-9pvn6m"
Unexpected error:
    <*errors.joinError | 0x6c70d0280d8>: 
    VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) timed out after 10m0.004s: DNS A record "*.e2e-dns-fs2vtm" not yet present in zone rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-fs2vtm' does not exist in resource group 'rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm
            correlation ID: 850b8d7d-f572-4c32-8b7c-4052d22cd119
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-9pvn6m"
Unexpected error:
    <*errors.joinError | 0x6c70d0280d8>: 
    VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud) timed out after 10m0.004s: DNS A record "*.e2e-dns-fs2vtm" not yet present in zone rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-fs2vtm' does not exist in resource group 'rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-kkv4f4xfm4kx-candidate-9pvn6m--managed/providers/Microsoft.Network/dnsZones/aro.k7l7c8r8l9b3j7g.kbo1.j5963776.hcp.osadev.cloud/A/*.e2e-dns-fs2vtm
            correlation ID: 850b8d7d-f572-4c32-8b7c-4052d22cd119
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-n5f5ld"
Unexpected error:
    <*errors.joinError | 0x29f2244004b0>: 
    VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record "*.e2e-dns-gkfmqf" not yet present in zone rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-gkfmqf' does not exist in resource group 'rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf
            correlation ID: d271f6f6-6cb4-472d-b6b5-9e1294932892
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-n5f5ld"
Unexpected error:
    <*errors.joinError | 0x29f2244004b0>: 
    VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record "*.e2e-dns-gkfmqf" not yet present in zone rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-gkfmqf' does not exist in resource group 'rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-tpslzg6rj8xj-candidate-n5f5ld--managed/providers/Microsoft.Network/dnsZones/aro.l9k5u3r1j0f0v1z.l4tl.j1933056.hcp.osadev.cloud/A/*.e2e-dns-gkfmqf
            correlation ID: d271f6f6-6cb4-472d-b6b5-9e1294932892
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-r5cltn"
Unexpected error:
    <*errors.joinError | 0x263db2a8cd50>: 
    VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record "*.e2e-dns-p2v629" not yet present in zone rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-p2v629' does not exist in resource group 'rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629
            correlation ID: 0eb3f5e1-8fdc-4f54-ad01-280d07fdef7f
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-r5cltn"
Unexpected error:
    <*errors.joinError | 0x263db2a8cd50>: 
    VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud) timed out after 10m0.003s: DNS A record "*.e2e-dns-p2v629" not yet present in zone rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-p2v629' does not exist in resource group 'rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6vt99sh5dqhr-candidate-r5cltn--managed/providers/Microsoft.Network/dnsZones/aro.u8t8x8m6x2i2q9i.jz74.j3603072.hcp.osadev.cloud/A/*.e2e-dns-p2v629
            correlation ID: 0eb3f5e1-8fdc-4f54-ad01-280d07fdef7f
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-458kh7"
Unexpected error:
    <*errors.joinError | 0x1b0cdceb6450>: 
    VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record "*.e2e-dns-qg48vc" not yet present in zone rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-qg48vc' does not exist in resource group 'rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc
            correlation ID: 1332f9fe-dcac-4684-bfa0-1d5d13e9cf06
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-458kh7"
Unexpected error:
    <*errors.joinError | 0x1b0cdceb6450>: 
    VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) failed: VerifyIngressDNSRecord(aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud) timed out after 10m0.005s: DNS A record "*.e2e-dns-qg48vc" not yet present in zone rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-qg48vc' does not exist in resource group 'rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-m5tx8lx5j5bn-candidate-458kh7--managed/providers/Microsoft.Network/dnsZones/aro.g7r4k9t5x4l0l6s.4rfh.j1218048.hcp.osadev.cloud/A/*.e2e-dns-qg48vc
            correlation ID: 1332f9fe-dcac-4684-bfa0-1d5d13e9cf06
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-5-0-prdbsm"
Unexpected error:
    <*errors.joinError | 0xfaf21d3c618>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "cluster-np-prdbsm-dxj9f-p58c2": nodes "cluster-np-prdbsm-dxj9f-p58c2" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0xfaf223bc0a0>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0xfaf223bc080>{
                    msg: "failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0xfaf223bc060>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0xfaf21fc43c0>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "cluster-np-prdbsm-dxj9f-p58c2",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-5-0-prdbsm"
Unexpected error:
    <*errors.joinError | 0xfaf21d3c618>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "cluster-np-prdbsm-dxj9f-p58c2": nodes "cluster-np-prdbsm-dxj9f-p58c2" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0xfaf223bc0a0>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0xfaf223bc080>{
                    msg: "failed to label node cluster-np-prdbsm-dxj9f-p58c2 egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0xfaf223bc060>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"cluster-np-prdbsm-dxj9f-p58c2\": nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0xfaf21fc43c0>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"cluster-np-prdbsm-dxj9f-p58c2\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "cluster-np-prdbsm-dxj9f-p58c2",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster "cluster-hs-pre-546c5c" should complete its version rollout
Unexpected error:
    <*fmt.wrapError | 0x2e4bde720b40>: 
    VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39="Partial"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available
    {
        msg: "VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
        err: <*verifiers.pollObservationError | 0x2e4bde720ac0>{
            observation: <*errors.errorString | 0x2e4be03290f0>{
                s: "clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
            },
            interruption: <*fmt.wrapError | 0x2e4bde720aa0>{
                msg: "failed to get clusterversion \"version\": context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster "cluster-hs-pre-546c5c" should complete its version rollout
Unexpected error:
    <*fmt.wrapError | 0x2e4bde720b40>: 
    VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39="Partial"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available
    {
        msg: "VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
        err: <*verifiers.pollObservationError | 0x2e4bde720ac0>{
            observation: <*errors.errorString | 0x2e4be03290f0>{
                s: "clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available",
            },
            interruption: <*fmt.wrapError | 0x2e4bde720aa0>{
                msg: "failed to get clusterversion \"version\": context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:159]: cluster did not become visible in console.redhat.com
Unexpected error:
    <*fmt.wrapError | 0x1f4ce1090c80>: 
    VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id "462efae8-075b-4d74-bf89-a1bd7f9d3b6f": Get "https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1": can't get access token: invalid_grant: Invalid refresh token
    {
        msg: "VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1\": can't get access token: invalid_grant: Invalid refresh token",
        err: <*fmt.wrapError | 0x1f4ce110ccc0>{
            msg: "failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1\": can't get access token: invalid_grant: Invalid refresh token",
            err: <*url.Error | 0x1f4ce0ce8a80>{
                Op: "Get",
                URL: "https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1",
                Err: <*fmt.wrapError | 0x1f4ce110cca0>{
                    msg: "can't get access token: invalid_grant: Invalid refresh token",
                    err: <*errors.errorString | 0x1f4ce1105790>{
                        s: "invalid_grant: Invalid refresh token",
                    },
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:159]: cluster did not become visible in console.redhat.com
Unexpected error:
    <*fmt.wrapError | 0x1f4ce1090c80>: 
    VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id "462efae8-075b-4d74-bf89-a1bd7f9d3b6f": Get "https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1": can't get access token: invalid_grant: Invalid refresh token
    {
        msg: "VerifyClusterVisibleInConsole timed out after 20m0.02s: failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1\": can't get access token: invalid_grant: Invalid refresh token",
        err: <*fmt.wrapError | 0x1f4ce110ccc0>{
            msg: "failed to query OCM clusters API for external_id \"462efae8-075b-4d74-bf89-a1bd7f9d3b6f\": Get \"https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1\": can't get access token: invalid_grant: Invalid refresh token",
            err: <*url.Error | 0x1f4ce0ce8a80>{
                Op: "Get",
                URL: "https://api.openshift.com/api/clusters_mgmt/v1/clusters?search=external_id+%3D+%27462efae8-075b-4d74-bf89-a1bd7f9d3b6f%27&size=1",
                Err: <*fmt.wrapError | 0x1f4ce110cca0>{
                    msg: "can't get access token: invalid_grant: Invalid refresh token",
                    err: <*errors.errorString | 0x1f4ce1105790>{
                        s: "invalid_grant: Invalid refresh token",
                    },
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:117]: Insights Operator was not deployed or did not reach Disabled/NoToken state
Unexpected error:
    <*fmt.wrapError | 0xa7745440900>: 
    VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator "insights" has no Disabled condition yet ()
    {
        msg: "VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()",
        err: <*verifiers.pollObservationError | 0xa77454408e0>{
            observation: <*errors.errorString | 0xa77456c4580>{
                s: "ClusterOperator \"insights\" has no Disabled condition yet ()",
            },
            interruption: <*fmt.wrapError | 0xa77454408c0>{
                msg: "failed to get ClusterOperator \"insights\": context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_insights_operator.go:117]: Insights Operator was not deployed or did not reach Disabled/NoToken state
Unexpected error:
    <*fmt.wrapError | 0xa7745440900>: 
    VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator "insights" has no Disabled condition yet ()
    {
        msg: "VerifyInsightsOperatorDisabledNoToken timed out after 10m0.005s: ClusterOperator \"insights\" has no Disabled condition yet ()",
        err: <*verifiers.pollObservationError | 0xa77454408e0>{
            observation: <*errors.errorString | 0xa77456c4580>{
                s: "ClusterOperator \"insights\" has no Disabled condition yet ()",
            },
            interruption: <*fmt.wrapError | 0xa77454408c0>{
                msg: "failed to get ClusterOperator \"insights\": context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2874138e2000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*errors.errorString | 0x2874134f7c90>{
            s: "dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2874138e2000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-w7kwr6bpq57t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*errors.errorString | 0x2874134f7c90>{
            s: "dev/centralus/ci01-j4637312-mgmt-2/ocm-arohcpci01-2tn2f6qu0skm7r1cd4s42o9104lkbkll-m9o9d5o9m2l7k4w: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1623311283c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x1623329ba440>{
            s: "dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1623311283c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x1623329ba440>{
            s: "dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2424493382e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x2424493345b0>{
            s: "dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2424493382e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hcf7x894jgzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x2424493345b0>{
            s: "dev/centralus/ci01-j0183936-mgmt-1/ocm-arohcpci01-2to1b0ie1a4878pqeni53tn93ag3ur6i-t8h4i5b2e7a9g5f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x31615201e9a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
        err: <*verifiers.pollObservationError | 0x31615201e980>{
            observation: <*errors.errorString | 0x316153080da0>{
                s: "no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
            },
            interruption: <*fmt.wrapError | 0x31615201e960>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j9822336-mgmt-1\"; let ns = \"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x316150f60aa0>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x31615201e920>{
                        msg: "With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j9822336-mgmt-1\"; let ns = \"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x316152c10a80>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x31615201e9a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-7qgq6lcld5tj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
        err: <*verifiers.pollObservationError | 0x31615201e980>{
            observation: <*errors.errorString | 0x316153080da0>{
                s: "no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
            },
            interruption: <*fmt.wrapError | 0x31615201e960>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j9822336-mgmt-1\"; let ns = \"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x316150f60aa0>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x31615201e920>{
                        msg: "With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j9822336-mgmt-1\"; let ns = \"ocm-arohcpci01-2tnvqtddfc0hnmn52ol3g2lfbm6i22ri-i7y4b4h2z7o4a5h\"; let start = datetime(2026-10-28T19:21:43.9974261Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x316152c10a80>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xd26a6364d80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
        err: <*verifiers.pollObservationError | 0xd26a6364d60>{
            observation: <*errors.errorString | 0xd26a57e61b0>{
                s: "no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
            },
            interruption: <*fmt.wrapError | 0xd26a6364d40>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"centralus\"; let cl = \"ci01-j6172544-mgmt-2\"; let ns = \"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0xd26a5820dc0>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0xd26a6364d20>{
                        msg: "With query: let env = \"dev\"; let rgn = \"centralus\"; let cl = \"ci01-j6172544-mgmt-2\"; let ns = \"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0xd26a719e570>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xd26a6364d80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8jc7r25jfq6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
        err: <*verifiers.pollObservationError | 0xd26a6364d60>{
            observation: <*errors.errorString | 0xd26a57e61b0>{
                s: "no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
            },
            interruption: <*fmt.wrapError | 0xd26a6364d40>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"centralus\"; let cl = \"ci01-j6172544-mgmt-2\"; let ns = \"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0xd26a5820dc0>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0xd26a6364d20>{
                        msg: "With query: let env = \"dev\"; let rgn = \"centralus\"; let cl = \"ci01-j6172544-mgmt-2\"; let ns = \"ocm-arohcpci01-2tnpie08b6e3m8ktaflq60p9p59bkmrn-q5i5v4f4b9e4k5j\"; let start = datetime(2026-10-28T12:14:00.5191321Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0xd26a719e570>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3756ccac8820>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z",
        err: <*errors.errorString | 0x3756cb62d010>{
            s: "dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3756ccac8820>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dx26gclbnqgv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z",
        err: <*errors.errorString | 0x3756cb62d010>{
            s: "dev/westus3/ci01-j5676416-mgmt-2/ocm-arohcpci01-2tnufqipa4peho2pme6jiu9n7kj695rj-r8q8e3h9t3h8q1o router=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000: expected matching summary after test start and within 8m; summaryUID=8004b169-b285-4616-88a5-3129e6cda029 node=aks-userswft2-37523794-vmss000000 pass=5eb81438-b4f4-4d92-b379-a83081a8b2cc at=2026-10-28T18:18:30Z",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2680bac6c1a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
        err: <*verifiers.pollObservationError | 0x2680bac6c180>{
            observation: <*errors.errorString | 0x2680b95b3860>{
                s: "no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
            },
            interruption: <*fmt.wrapError | 0x2680bac6c160>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j1816448-mgmt-2\"; let ns = \"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x2680b9c4c190>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x2680bac6c140>{
                        msg: "With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j1816448-mgmt-2\"; let ns = \"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x2680bb4eaab0>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2680bac6c1a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9xwk6zxxz7m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
        err: <*verifiers.pollObservationError | 0x2680bac6c180>{
            observation: <*errors.errorString | 0x2680b95b3860>{
                s: "no customer resource ID -> live ReadDesire HostedCluster -> fleet management cluster mapping in Kusto",
            },
            interruption: <*fmt.wrapError | 0x2680bac6c160>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j1816448-mgmt-2\"; let ns = \"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x2680b9c4c190>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x2680bac6c140>{
                        msg: "With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j1816448-mgmt-2\"; let ns = \"ocm-arohcpci01-2tmt6fqch3btra8cfaickj0t85ovobt1-d4f6x6g2r2p5t9d\"; let start = datetime(2026-10-27T03:57:03.7385865Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x2680bb4eaab0>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x910473e2000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x91047780d20>{
            s: "dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x910473e2000>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spr5zsf6h48p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x91047780d20>{
            s: "dev/westus3/ci01-j5971328-mgmt-2/ocm-arohcpci01-2tne6fmsv15g1h9j1vkcqdkjq8bfs0ld-cluster: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x22cffa244a80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x22cffa418620>{
            s: "dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x22cffa244a80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*errors.errorString | 0x22cffa418620>{
            s: "dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2b13ff3f4aa0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*errors.errorString | 0x2b13ffa1a410>{
            s: "dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2b13ff3f4aa0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8qcj8gl78j5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*errors.errorString | 0x2b13ffa1a410>{
            s: "dev/westus3/ci01-j2262528-mgmt-1/ocm-arohcpci01-2tnnatgpnn1ff4v4ufusun418oclrbge-r6d7p9b2f9o5i3m: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2555ba2ee820>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*verifiers.pollObservationError | 0x2555ba2ee800>{
            observation: <*errors.errorString | 0x2555b9ce7460>{
                s: "dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
            },
            interruption: <*fmt.wrapError | 0x2555ba2ee7e0>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j7087616-mgmt-2\"; let ns = \"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x2555b95a8370>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x2555ba2ee7c0>{
                        msg: "With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j7087616-mgmt-2\"; let ns = \"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x2555baea86f0>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2555ba2ee820>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-95ktkd6tw5dw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
        err: <*verifiers.pollObservationError | 0x2555ba2ee800>{
            observation: <*errors.errorString | 0x2555b9ce7460>{
                s: "dev/westus3/ci01-j7087616-mgmt-2/ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f: expected all current routers >= desired replicas > 0; desired=3 current=2 rows=2",
            },
            interruption: <*fmt.wrapError | 0x2555ba2ee7e0>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j7087616-mgmt-2\"; let ns = \"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x2555b95a8370>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x2555ba2ee7c0>{
                        msg: "With query: let env = \"dev\"; let rgn = \"westus3\"; let cl = \"ci01-j7087616-mgmt-2\"; let ns = \"ocm-arohcpci01-2tn11bbfhosljf97vunimgc88lhlamlu-u6q6o6h9w2m2h0f\"; let start = datetime(2026-10-27T08:19:08.7156738Z);\nlet scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and environment == env and region == rgn and cluster == cl and namespace == ns\n| where objectKind in (\"Pod\", \"Deployment\")\n| summarize arg_max(timestamp, *) by objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project UID = uid, Node = tostring(object.spec.nodeName);\nlet desired = toscalar(inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project tolong(object.spec.replicas));\nlet current = toscalar(pods | count);\nlet summaries = containerLogs\n| where timestamp >= start and environment == env and region == rgn and cluster == cl\n| where container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| where tostring(log.pod_namespace) == ns\n| extend UID = tostring(log.pod_uid)\n| summarize arg_max(timestamp, *) by UID\n| project UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\npods\n| join kind=leftouter summaries on UID\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x2555baea86f0>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-dj4ptk"
Unexpected error:
    <*errors.joinError | 0xd5290aa85b8>: 
    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is "Pending", expected Bound; pod phase is "Pending"
    [storageclass] name=managed-csi provisioner=disk.csi.azure.com
    [pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi
    [pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=
    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded
    [events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim "e2e-csi-disk-8ql64/e2e-csi-pvc" (reason=Provisioning, count=11, age=10s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)
    {
        errs: [
            <*fmt.wrapError | 0xd5290a963a0>{
                msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)",
                err: <*fmt.wrapError | 0xd5290356ce0>{
                    msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)",
                    err: <*verifiers.pollObservationError | 0xd5290356540>{
                        observation: <*errors.errorString | 0xd52904125d0>{
                            s: "PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"",
                        },
                        interruption: <*fmt.wrapError | 0xd5290356520>{
                            msg: "failed to get PVC e2e-csi-disk-8ql64/e2e-csi-pvc: context deadline exceeded",
                            err: <context.deadlineExceededError>{},
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-dj4ptk"
Unexpected error:
    <*errors.joinError | 0xd5290aa85b8>: 
    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is "Pending", expected Bound; pod phase is "Pending"
    [storageclass] name=managed-csi provisioner=disk.csi.azure.com
    [pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi
    [pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=
    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded
    [events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim "e2e-csi-disk-8ql64/e2e-csi-pvc" (reason=Provisioning, count=11, age=10s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)
    [events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)
    {
        errs: [
            <*fmt.wrapError | 0xd5290a963a0>{
                msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)",
                err: <*fmt.wrapError | 0xd5290356ce0>{
                    msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0.001s: PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-8ql64/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-8ql64/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-8ql64 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-8ql64/e2e-csi-pvc\" (reason=Provisioning, count=11, age=10s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node c9x9s6y3p8f9j2i-np-dj4ptk-xtdjf-gwssm (reason=ProvisioningFailed, count=7, age=8m44s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m32s)\n[events] e2e-csi-disk-8ql64 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=3, age=10s)",
                    err: <*verifiers.pollObservationError | 0xd5290356540>{
                        observation: <*errors.errorString | 0xd52904125d0>{
                            s: "PVC e2e-csi-disk-8ql64/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"",
                        },
                        interruption: <*fmt.wrapError | 0xd5290356520>{
                            msg: "failed to get PVC e2e-csi-disk-8ql64/e2e-csi-pvc: context deadline exceeded",
                            err: <context.deadlineExceededError>{},
                        },
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-6lzc2b"
Unexpected error:
    <*errors.joinError | 0x263fb059e4c8>: 
    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is "Pending", expected Bound; pod phase is "Pending"
    [storageclass] name=managed-csi provisioner=disk.csi.azure.com
    [pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi
    [pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=
    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded
    [events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim "e2e-csi-disk-sfxr9/e2e-csi-pvc" (reason=Provisioning, count=10, age=2m54s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)
    {
        errs: [
            <*fmt.wrapError | 0x263fb0784660>{
                msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)",
                err: <*fmt.wrapError | 0x263fb09fc440>{
                    msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)",
                    err: <*errors.errorString | 0x263fb0ff0b50>{
                        s: "PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"",
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-23-6lzc2b"
Unexpected error:
    <*errors.joinError | 0x263fb059e4c8>: 
    VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is "Pending", expected Bound; pod phase is "Pending"
    [storageclass] name=managed-csi provisioner=disk.csi.azure.com
    [pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi
    [pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=
    [pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded
    [events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin "VolumeBinding": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim "e2e-csi-disk-sfxr9/e2e-csi-pvc" (reason=Provisioning, count=10, age=2m54s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)
    [events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)
    {
        errs: [
            <*fmt.wrapError | 0x263fb0784660>{
                msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) failed: VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)",
                err: <*fmt.wrapError | 0x263fb09fc440>{
                    msg: "VerifyCSIPersistentVolume(disk.csi.azure.com) timed out after 10m0s: PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"\n[storageclass] name=managed-csi provisioner=disk.csi.azure.com\n[pvc] e2e-csi-disk-sfxr9/e2e-csi-pvc phase=Pending volumeName= storageClass=managed-csi\n[pod] e2e-csi-disk-sfxr9/e2e-csi-pod phase=Pending reason= message=\n[pod] condition type=PodScheduled status=False reason=SchedulerError message=running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded\n[events] e2e-csi-disk-sfxr9 Pod/e2e-csi-pod: running PreBind plugin \"VolumeBinding\": binding volumes: context deadline exceeded (reason=FailedScheduling, count=0, age=0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: waiting for first consumer to be created before binding (reason=WaitForFirstConsumer, count=1, age=10m0s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=3, age=9m57s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: External provisioner is provisioning volume for claim \"e2e-csi-disk-sfxr9/e2e-csi-pvc\" (reason=Provisioning, count=10, age=2m54s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: error generating accessibility requirements: no topology key found for node n2t7h9a5i9i7j1l-np-6lzc2b-lzl2m-btqtb (reason=ProvisioningFailed, count=6, age=9m29s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: Waiting for a volume to be created either by the external provisioner 'disk.csi.azure.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. (reason=ExternalProvisioning, count=26, age=3m19s)\n[events] e2e-csi-disk-sfxr9 PersistentVolumeClaim/e2e-csi-pvc: rpc error: code = DeadlineExceeded desc = context deadline exceeded (reason=ProvisioningFailed, count=4, age=2m24s)",
                    err: <*errors.errorString | 0x263fb0ff0b50>{
                        s: "PVC e2e-csi-disk-sfxr9/e2e-csi-pvc phase is \"Pending\", expected Bound; pod phase is \"Pending\"",
                    },
                },
            },
        ],
    }
occurred

time=2026-10-27T15:30:56.670Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description="Step cluster-create\n  Kind: Shell\n  Command: ./aks-cluster-create\n"
time=2026-10-27T16:00:56.129Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err="stamp 1: error running Shell Step, failed to execute shell command: {\"time\":\"2026-10-27T15:30:56.681953143Z\",\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5601792-mgmt-1\",\"clusterName\":\"ci01-j5601792-mgmt-1\",\"region\":\"centralus\"}\n{\"time\":\"2026-10-27T15:30:58.731109745Z\",\"level\":\"INFO\",\"msg\":\"configured pools\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5601792-mgmt-1\",\"clusterName\":\"ci01-j5601792-mgmt-1\",\"region\":\"centralus\",\"system\":1,\"worker\":3,\"infra\":2}\n{\"time\":\"2026-10-27T15:30:58.873423526Z\",\"level\":\"INFO\",\"msg\":\"deploying configured resources\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5601792-mgmt-1\",\"clusterName\":\"ci01-j5601792-mgmt-1\",\"region\":\"centralus\",\"deployment\":\"aks-cluster-create\",\"poolCount\":6,\"clusterFields\":null}\n signal: killed"

fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal
Unexpected error:
    <*fmt.wrapError | 0x1b303bad2a00>: 
    failed to create service principal: create service principal for appId "a52f84ec-ecc1-43ca-9731-aac8f9f81ad5": Post "https://graph.microsoft.com/v1.0/servicePrincipals": net/http: request canceled (Client.Timeout exceeded while awaiting headers)
    {
        msg: "failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
        err: <*fmt.wrapError | 0x1b303bad2980>{
            msg: "create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
            err: <*url.Error | 0x1b303c6b42d0>{
                Op: "Post",
                URL: "https://graph.microsoft.com/v1.0/servicePrincipals",
                Err: <*http.timeoutError | 0x1b303d466f50>{
                    err: "net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal
Unexpected error:
    <*fmt.wrapError | 0x1b303bad2a00>: 
    failed to create service principal: create service principal for appId "a52f84ec-ecc1-43ca-9731-aac8f9f81ad5": Post "https://graph.microsoft.com/v1.0/servicePrincipals": net/http: request canceled (Client.Timeout exceeded while awaiting headers)
    {
        msg: "failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
        err: <*fmt.wrapError | 0x1b303bad2980>{
            msg: "create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
            err: <*url.Error | 0x1b303c6b42d0>{
                Op: "Post",
                URL: "https://graph.microsoft.com/v1.0/servicePrincipals",
                Err: <*http.timeoutError | 0x1b303d466f50>{
                    err: "net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
                },
            },
        },
    }
occurred

time=2026-10-28T17:01:13.002Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description="Step deploy\n Kind: Helm\n"
time=2026-10-28T17:01:13.550Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values="map[azureArmHelperIdentityCertName:armHelperPoolCert-25 azureArmHelperIdentityClientId:ea5e9c6c-1933-464d-8bf6-db343c22b92d azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:ae6cf536-acc0-414c-ada8-f268aeeb97e3 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-17 azureMiMockServicePrincipalClientId:e7112f7f-5b45-480e-bc44-abf9f219e6f0 azureMiMockServicePrincipalPrincipalId:39eefc4f-5e24-491d-9f09-bcc0ad573bcf azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j3195648 cosmosDatabaseUrl:https://arohcpci01-rp-j3195648.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:b6e44eb348113224aa292d06b6cbf4154e881fe0a8617e245e7c6555f046d495] imageDigest:sha256:b6e44eb348113224aa292d06b6cbf4154e881fe0a8617e245e7c6555f046d495 imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi monitoringApiGroup:monitoring.coreos.com ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj3195648.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj3195648.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:a5d5ff83-799c-44fd-a535-8c4386588d02 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:canadacentral replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]"
time=2026-10-28T17:01:13.550Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy
time=2026-10-28T17:01:13.717Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service
time=2026-10-28T17:01:13.899Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false
time=2026-10-28T17:01:13.899Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy
time=2026-10-28T17:01:13.952Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy
time=2026-10-28T17:11:19.322Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err="resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\ncontext deadline exceeded"
time=2026-10-28T17:11:19.322Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err="error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 2/3\ncontext deadline exceeded"

time=2026-10-28T00:24:38.589Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description="Step deploy\n Kind: Helm\n"
time=2026-10-28T00:24:39.214Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values="map[azure:map[clientId:80c01b85-e07b-47ba-b2c6-d25569ff69d4 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-ci01-maestro-j8270720.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j8270720] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]] zoneCount:3] fleet:map[namespace:fleet serviceAccount:fleet] image:map[digest:sha256:aba8ed3c2d6e8b0f6a5e17b2f65dba996c5cf673b73a35cc550521873b08c231 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] monitoringApiGroup:monitoring.coreos.com pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:d99261bb-4b6f-46d6-96d4-bae9c86ef284 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]"
time=2026-10-28T00:24:39.214Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy
time=2026-10-28T00:24:39.557Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro
time=2026-10-28T00:24:39.898Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false
time=2026-10-28T00:24:39.898Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy
time=2026-10-28T00:24:40.025Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy
time=2026-10-28T00:34:46.993Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err="resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\ncontext deadline exceeded"
time=2026-10-28T00:34:46.994Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err="error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: InProgress, message: Available: 0/2\ncontext deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.696s.
Expected success, but got an error:
    <*errors.errorString | 0x283ddd596200>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.696s.
Expected success, but got an error:
    <*errors.errorString | 0x283ddd596200>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=10944, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0xca1fe06c6d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0xca1fe06c6d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11333, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0xe6261437860>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0xe6261437860>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.
Expected success, but got an error:
    <*errors.errorString | 0x3952e9095e40>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.
Expected success, but got an error:
    <*errors.errorString | 0x3952e9095e40>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0x867032a97d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0x867032a97d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=14276, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.
Expected success, but got an error:
    <*errors.errorString | 0x20bd1e04e7a0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.
Expected success, but got an error:
    <*errors.errorString | 0x20bd1e04e7a0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7034, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.
Expected success, but got an error:
    <*errors.errorString | 0x2de4f1b0a5d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.
Expected success, but got an error:
    <*errors.errorString | 0x2de4f1b0a5d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=7169, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 603.481s.
Expected success, but got an error:
    <*errors.errorString | 0x12a5a347cf40>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 603.481s.
Expected success, but got an error:
    <*errors.errorString | 0x12a5a347cf40>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.002s.
Expected success, but got an error:
    <*errors.errorString | 0x211f21252720>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.002s.
Expected success, but got an error:
    <*errors.errorString | 0x211f21252720>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8412, acm-agent=0",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.
Expected success, but got an error:
    <*errors.errorString | 0x294c3a4e6f50>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.
Expected success, but got an error:
    <*errors.errorString | 0x294c3a4e6f50>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0",
    }

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-28T20:00:50Z
  Ended:       2026-10-28T20:06:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j5001984-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tnvsms9l35e1dde1cr2bnekd5epm5j9-ingress-cert-le", pod="openshift-apiserver-7cb4d98fc-rzx96", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tnvsms9l35e1dde1cr2bnekd5epm5j9-ingress-cert-le/openshift-apiserver-7cb4d98fc-rzx96 has been in a non-ready state for longer than 5 minutes.

alert fired 2 time(s) (1 unknown, 1 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-27T23:40:09Z
  Ended:       2026-10-27T23:57:09Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1933056-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tneda066vj6kdgdjqh3ub42gu0lckg9-ephemeral-disk", pod="etcd-0", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tneda066vj6kdgdjqh3ub42gu0lckg9-ephemeral-disk/etcd-0 has been in a non-ready state for longer than 5 minutes.

alert fired 13 time(s) (12 unknown, 1 known)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:44:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn75avtj8u4futjt2om8gigffgddu35-ingress-cert-le", pod="openshift-apiserver-5fbf969589-rwzww", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn75avtj8u4futjt2om8gigffgddu35-ingress-cert-le/openshift-apiserver-5fbf969589-rwzww has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:44:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn74o7qhu6go8i6sdevboooi4cvm5o8-o9s7l1c0u6a0c2b", pod="openshift-apiserver-7c75f5b8c-869mc", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn74o7qhu6go8i6sdevboooi4cvm5o8-o9s7l1c0u6a0c2b/openshift-apiserver-7c75f5b8c-869mc has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:45:13Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn75q9b8j8s3323r8hbovnl3fa3ah93-a8h6t7r0k6i7a1w", pod="openshift-apiserver-6cf589c77c-9cn7v", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn75q9b8j8s3323r8hbovnl3fa3ah93-a8h6t7r0k6i7a1w/openshift-apiserver-6cf589c77c-9cn7v has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:45:13Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn74rifog60es9rvt2t2mfrpvs36h9e-t2y6g6x3g9p9s5t", pod="openshift-apiserver-9c9cfdf89-62q4b", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn74rifog60es9rvt2t2mfrpvs36h9e-t2y6g6x3g9p9s5t/openshift-apiserver-9c9cfdf89-62q4b has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:45:13Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn757i0j3orr665p0dcqg487jcvp9a9-q1n4k4z8u2y9x7u", pod="openshift-apiserver-7bbccfbcd8-nj4zh", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn757i0j3orr665p0dcqg487jcvp9a9-q1n4k4z8u2y9x7u/openshift-apiserver-7bbccfbcd8-nj4zh has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:45:13Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn75637s7ka00dgh3b0o5nl473kscrl-h2n8s4a0l1b6e9x", pod="openshift-apiserver-7b79cf8d6d-klztg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn75637s7ka00dgh3b0o5nl473kscrl-h2n8s4a0l1b6e9x/openshift-apiserver-7b79cf8d6d-klztg has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:45:13Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn7522f0rjmsl54mp9vum53bfeis22t-x1e5u3y9j2l5a2u", pod="openshift-apiserver-7d75b894c6-n4cmx", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn7522f0rjmsl54mp9vum53bfeis22t-x1e5u3y9j2l5a2u/openshift-apiserver-7d75b894c6-n4cmx has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:46:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn75ndq1oh2entq0cv9ulrug5jog853-ingress-cert-le", pod="openshift-apiserver-764b69ffcb-7ngbg", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn75ndq1oh2entq0cv9ulrug5jog853-ingress-cert-le/openshift-apiserver-764b69ffcb-7ngbg has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:46:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn7cebrepv6hitmognobm7c9cl2clnd-o5c0v9b6h0c5z0a", pod="cluster-policy-controller-698bfd9c65-v6xdj", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn7cebrepv6hitmognobm7c9cl2clnd-o5c0v9b6h0c5z0a/cluster-policy-controller-698bfd9c65-v6xdj has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:46:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn75h85al80vo23t362mjtc3qgd04dp-c1b6v6c2x7c8p8f", pod="openshift-apiserver-57bf4dc46f-h7mj5", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn75h85al80vo23t362mjtc3qgd04dp-c1b6v6c2x7c8p8f/openshift-apiserver-57bf4dc46f-h7mj5 has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:46:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn759i4ns4omt60u9uecngkrn5mj618-u8q4i0h7g0i1r6k", pod="openshift-apiserver-59b86f6c67-76jwk", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn759i4ns4omt60u9uecngkrn5mj618-u8q4i0h7g0i1r6k/openshift-apiserver-59b86f6c67-76jwk has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:47:14Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j1476864-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tn7615u8n39pf897sksqb5f7beb9d2u-b0o3o9t7e5r0u9g", pod="openshift-apiserver-77c77b89b8-kr9pl", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tn7615u8n39pf897sksqb5f7beb9d2u-b0o3o9t7e5r0u9g/openshift-apiserver-77c77b89b8-kr9pl has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster "private-kas" with private KAS
Unexpected error:
    <*fmt.wrapError | 0x3a2a28c8a260>: 
    failed to create HCP cluster private-kas: failed waiting for cluster="private-kas" in resourcegroup="private-kas-h8dst6v865z6" to finish creating: GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff",
      "name": "455ab4ea-ab03-4a58-91c6-5ad5406405ff",
      "status": "Failed",
      "startTime": "2026-10-28T20:16:02.73250898Z",
      "endTime": "2026-10-28T20:45:04.251585998Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff
            correlation ID: 582a61df-2172-42e6-962b-b544533d3da6
            client request ID: a46122fb-f2eb-46f8-9be1-73dd5a442cce
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster "private-kas" with private KAS
Unexpected error:
    <*fmt.wrapError | 0x3a2a28c8a260>: 
    failed to create HCP cluster private-kas: failed waiting for cluster="private-kas" in resourcegroup="private-kas-h8dst6v865z6" to finish creating: GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff",
      "name": "455ab4ea-ab03-4a58-91c6-5ad5406405ff",
      "status": "Failed",
      "startTime": "2026-10-28T20:16:02.73250898Z",
      "endTime": "2026-10-28T20:45:04.251585998Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1157504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/455ab4ea-ab03-4a58-91c6-5ad5406405ff
            correlation ID: 582a61df-2172-42e6-962b-b544533d3da6
            client request ID: a46122fb-f2eb-46f8-9be1-73dd5a442cce
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster "cidr-connectivity-test" with authorized CIDRs
Unexpected error:
    <*fmt.wrapError | 0x370d5101eb20>: 
    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-tgjxrh8mcrw6" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb",
      "name": "cce1648d-4b31-48bb-9884-f170bb5391eb",
      "status": "Failed",
      "startTime": "2026-10-28T19:28:55.409225524Z",
      "endTime": "2026-10-28T19:58:03.222551854Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb
            correlation ID: d6e5235d-8c3c-4404-82e9-f3e3b3165e1e
            client request ID: 4d040942-ab41-4fbc-8d6e-d69a84105ef1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster "cidr-connectivity-test" with authorized CIDRs
Unexpected error:
    <*fmt.wrapError | 0x370d5101eb20>: 
    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-tgjxrh8mcrw6" to finish creating: GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb",
      "name": "cce1648d-4b31-48bb-9884-f170bb5391eb",
      "status": "Failed",
      "startTime": "2026-10-28T19:28:55.409225524Z",
      "endTime": "2026-10-28T19:58:03.222551854Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8717568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce1648d-4b31-48bb-9884-f170bb5391eb
            correlation ID: d6e5235d-8c3c-4404-82e9-f3e3b3165e1e
            client request ID: 4d040942-ab41-4fbc-8d6e-d69a84105ef1
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jl2sj8 with version 4.21.40
Unexpected error:
    <*fmt.wrapError | 0x2ee0c1834d20>: 
    failed to create HCP cluster np-version-upgrade-cluster-jl2sj8: failed waiting for cluster="np-version-upgrade-cluster-jl2sj8" in resourcegroup="rg-np-version-upgrade-jl2sj8-22vp9qvqkmcf" to finish creating: GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583",
      "name": "267d3daf-f468-46eb-af3a-f1b72d573583",
      "status": "Failed",
      "startTime": "2026-10-28T19:42:27.649991944Z",
      "endTime": "2026-10-28T20:11:28.996573337Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583
            correlation ID: c45a1c96-f9ee-44e0-8639-c952cdb59a23
            client request ID: fe15252c-edf3-4b0e-8aaa-b5cd06393883
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jl2sj8 with version 4.21.40
Unexpected error:
    <*fmt.wrapError | 0x2ee0c1834d20>: 
    failed to create HCP cluster np-version-upgrade-cluster-jl2sj8: failed waiting for cluster="np-version-upgrade-cluster-jl2sj8" in resourcegroup="rg-np-version-upgrade-jl2sj8-22vp9qvqkmcf" to finish creating: GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583",
      "name": "267d3daf-f468-46eb-af3a-f1b72d573583",
      "status": "Failed",
      "startTime": "2026-10-28T19:42:27.649991944Z",
      "endTime": "2026-10-28T20:11:28.996573337Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9822336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/267d3daf-f468-46eb-af3a-f1b72d573583
            correlation ID: c45a1c96-f9ee-44e0-8639-c952cdb59a23
            client request ID: fe15252c-edf3-4b0e-8aaa-b5cd06393883
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-28T20:39:20Z
  Ended:       2026-10-28T21:05:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j1157504-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.172:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="777a1913-78dc-4e11-9844-4f50d6675327"
  Description: Pod ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-28T20:40:18Z
  Ended:       2026-10-28T21:06:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j1157504-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.172:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="777a1913-78dc-4e11-9844-4f50d6675327"
  Description: Pod ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-28T19:50:44Z
  Ended:       2026-10-28T20:18:45Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j8717568-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.254:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="aad698ce-6b47-41d1-8ddd-e6e4cb15b5d7"
  Description: Pod ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-28T19:50:44Z
  Ended:       2026-10-28T20:19:45Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j8717568-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.254:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="aad698ce-6b47-41d1-8ddd-e6e4cb15b5d7"
  Description: Pod ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-28T12:59:31Z
  Ended:       2026-10-28T13:28:30Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j6945280-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.108:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="5496ebfd-2ab4-43b8-9f9a-e0483581f4bd"
  Description: Pod ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-28T13:00:33Z
  Ended:       2026-10-28T13:27:30Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j6945280-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.108:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="5496ebfd-2ab4-43b8-9f9a-e0483581f4bd"
  Description: Pod ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-28T20:52:20Z
  Ended:       2026-10-28T21:03:23Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j1157504-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.172:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-28T20:52:20Z
  Ended:       2026-10-28T21:03:23Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j1157504-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.172:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1157504/providers/microsoft.monitor/accounts/hcps-j1157504", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-p48cd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2to0k0skbecjdmuid7733ieqr7h7trvg-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-28T20:03:50Z
  Ended:       2026-10-28T20:15:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j8717568-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.254:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-28T20:03:50Z
  Ended:       2026-10-28T20:15:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j8717568-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.254:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8717568/providers/microsoft.monitor/accounts/hcps-j8717568", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-59dhx", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tnvtu6q95ufl5426ec9nsjbnsqfa0aa-a8y2r7f6q8h7z0t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-28T13:12:32Z
  Ended:       2026-10-28T13:28:32Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6945280-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.108:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6945280/providers/microsoft.monitor/accounts/hcps-j6945280", microsoft.metrics_include_label="service", namespace="ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-fct9q", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tnptam9khdl3svuvm4seuq5gr0ieit6-v5j4g5d3s9p2z6k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x2db4fc89320>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-zbnm2gmmrc7k" to finish creating: GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b",
      "name": "13b873cd-2efa-4559-ab15-ff37ab2d031b",
      "status": "Failed",
      "startTime": "2026-10-28T16:52:57.688694522Z",
      "endTime": "2026-10-28T17:12:06.752450543Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b
            correlation ID: 596c64f1-beb7-4366-a3b9-24ac7fe089dc
            client request ID: 7b2dfc27-0f04-4e6d-a8a1-787b24c6e549
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x2db4fc89320>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-zbnm2gmmrc7k" to finish creating: GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b",
      "name": "13b873cd-2efa-4559-ab15-ff37ab2d031b",
      "status": "Failed",
      "startTime": "2026-10-28T16:52:57.688694522Z",
      "endTime": "2026-10-28T17:12:06.752450543Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine j8c0h2z2t3o8e9b-np-delete-main-69xnv-msqbr: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8443648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b873cd-2efa-4559-ab15-ff37ab2d031b
            correlation ID: 596c64f1-beb7-4366-a3b9-24ac7fe089dc
            client request ID: 7b2dfc27-0f04-4e6d-a8a1-787b24c6e549

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0xb71d4fca080>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-bqbx2znvzb54" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9",
      "name": "0ed006bd-b8aa-4770-9062-3fd090d4fea9",
      "status": "Failed",
      "startTime": "2026-10-27T23:25:38.076776565Z",
      "endTime": "2026-10-27T23:44:40.197052822Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-xjlrw-mjpj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xjlrw-mjpj4 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-xjlrw-7jnb9: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-bqbx2znvzb54--managed/ingress-cert-le-np-1-xjlrw-7jnb9-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-xjlrw-7jnb9-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9
            correlation ID: d0096963-5f3a-4ca8-942f-d516adbeb2fc
            client request ID: 336f75bd-67db-4bb4-b035-81a9154977bb
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0xb71d4fca080>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-bqbx2znvzb54" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9",
      "name": "0ed006bd-b8aa-4770-9062-3fd090d4fea9",
      "status": "Failed",
      "startTime": "2026-10-27T23:25:38.076776565Z",
      "endTime": "2026-10-27T23:44:40.197052822Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-xjlrw-mjpj4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-xjlrw-mjpj4 to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-xjlrw-7jnb9: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-bqbx2znvzb54--managed/ingress-cert-le-np-1-xjlrw-7jnb9-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bqbx2znvzb54--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-xjlrw-7jnb9-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0ed006bd-b8aa-4770-9062-3fd090d4fea9
            correlation ID: d0096963-5f3a-4ca8-942f-d516adbeb2fc
            client request ID: 336f75bd-67db-4bb4-b035-81a9154977bb

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x3591356d60>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-6b928vvvc9lv" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331",
      "name": "e68fe219-929c-499f-9d94-52a89aa54331",
      "status": "Failed",
      "startTime": "2026-10-27T23:27:39.482004964Z",
      "endTime": "2026-10-27T23:46:40.23395218Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331
            correlation ID: 0fae4e0c-a402-45c6-8599-10c6e41f505d
            client request ID: c1da63b5-9f53-4e68-8989-fc0f3ef1f3cb
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x3591356d60>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-6b928vvvc9lv" to finish creating: GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331",
      "name": "e68fe219-929c-499f-9d94-52a89aa54331",
      "status": "Failed",
      "startTime": "2026-10-27T23:27:39.482004964Z",
      "endTime": "2026-10-27T23:46:40.23395218Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-qtp5t-vlfjh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-6b928vvvc9lv--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-qtp5t-vlfjh to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1933056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e68fe219-929c-499f-9d94-52a89aa54331
            correlation ID: 0fae4e0c-a402-45c6-8599-10c6e41f505d
            client request ID: c1da63b5-9f53-4e68-8989-fc0f3ef1f3cb

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-28T17:07:36Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3195648-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="clusters-service-64b56768b9-r7v9d", region="canadacentral", severity="warning"
  Description: Pod clusters-service/clusters-service-64b56768b9-r7v9d has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-28T17:07:36Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3195648-svc", component="kubernetes-infrastructure", namespace="fleet", pod="fleet-controller-7b9596cccb-g9thf", region="canadacentral", severity="warning"
  Description: Pod fleet/fleet-controller-7b9596cccb-g9thf has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-28T17:07:36Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3195648-svc", component="kubernetes-infrastructure", namespace="aro-hcp-admin-api", pod="admin-api-6d56468ffd-t9snj", region="canadacentral", severity="warning"
  Description: Pod aro-hcp-admin-api/admin-api-6d56468ffd-t9snj has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-28T17:07:36Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3195648-svc", component="kubernetes-infrastructure", namespace="aro-hcp-admin-api", pod="admin-api-6d56468ffd-rm24l", region="canadacentral", severity="warning"
  Description: Pod aro-hcp-admin-api/admin-api-6d56468ffd-rm24l has been in a non-ready state for longer than 5 minutes.

alert fired 21 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:19Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="aks-istio-system", pod="azure-service-mesh-istio-cni-addon-node-ck87t", region="westus3", severity="warning"
  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-ck87t has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="aks-secrets-store-csi-driver-n68qf", region="westus3", severity="warning"
  Description: Pod kube-system/aks-secrets-store-csi-driver-n68qf has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="ama-metrics-node-bx6p7", region="westus3", severity="warning"
  Description: Pod kube-system/ama-metrics-node-bx6p7 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="ama-metrics-node-6fggw", region="westus3", severity="warning"
  Description: Pod kube-system/ama-metrics-node-6fggw has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="eraser-aks-system-42508117-vmss000002-249kw", region="westus3", severity="warning"
  Description: Pod kube-system/eraser-aks-system-42508117-vmss000002-249kw has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="aks-istio-system", pod="azure-service-mesh-istio-cni-addon-node-f2bs8", region="westus3", severity="warning"
  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-f2bs8 has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="aks-secrets-store-csi-driver-pwgk9", region="westus3", severity="warning"
  Description: Pod kube-system/aks-secrets-store-csi-driver-pwgk9 has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="ama-metrics-node-sb9gv", region="westus3", severity="warning"
  Description: Pod kube-system/ama-metrics-node-sb9gv has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="eraser-aks-system-42508117-vmss000001-z2292", region="westus3", severity="warning"
  Description: Pod kube-system/eraser-aks-system-42508117-vmss000001-z2292 has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:20Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="aks-secrets-store-csi-driver-sgh48", region="westus3", severity="warning"
  Description: Pod kube-system/aks-secrets-store-csi-driver-sgh48 has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="aks-istio-system", pod="azure-service-mesh-istio-cni-addon-node-jkt84", region="westus3", severity="warning"
  Description: Pod aks-istio-system/azure-service-mesh-istio-cni-addon-node-jkt84 has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="eraser-aks-system-42508117-vmss000000-b6bzq", region="westus3", severity="warning"
  Description: Pod kube-system/eraser-aks-system-42508117-vmss000000-b6bzq has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:28:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="kube-system", pod="hubble-relay-58594b5dd7-p7rcj", region="westus3", severity="warning"
  Description: Pod kube-system/hubble-relay-58594b5dd7-p7rcj has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="sessiongate", pod="sessiongate-8696d46cbc-gcq7v", region="westus3", severity="warning"
  Description: Pod sessiongate/sessiongate-8696d46cbc-gcq7v has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="sessiongate", pod="sessiongate-8696d46cbc-7tl64", region="westus3", severity="warning"
  Description: Pod sessiongate/sessiongate-8696d46cbc-7tl64 has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="maestro", pod="maestro-6bc6657c6b-nc5dx", region="westus3", severity="warning"
  Description: Pod maestro/maestro-6bc6657c6b-nc5dx has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="clusters-service-567b46758d-6wzh8", region="westus3", severity="warning"
  Description: Pod clusters-service/clusters-service-567b46758d-6wzh8 has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="clusters-service-567b46758d-rsmsf", region="westus3", severity="warning"
  Description: Pod clusters-service/clusters-service-567b46758d-rsmsf has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="ocm-cs-db-7594d5ccfc-vd2b7", region="westus3", severity="warning"
  Description: Pod clusters-service/ocm-cs-db-7594d5ccfc-vd2b7 has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="clusters-service", pod="clusters-service-567b46758d-hzqfx", region="westus3", severity="warning"
  Description: Pod clusters-service/clusters-service-567b46758d-hzqfx has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Fired (not resolved)
  Started:     2026-10-28T00:30:21Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8270720-svc", component="kubernetes-infrastructure", namespace="maestro", pod="maestro-6bc6657c6b-stp2q", region="westus3", severity="warning"
  Description: Pod maestro/maestro-6bc6657c6b-stp2q has been in a non-ready state for longer than 5 minutes.

alert fired 8 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="ocm-controller-859484fc49-qhntv", region="westus3", severity="warning"
  Description: Pod multicluster-engine/ocm-controller-859484fc49-qhntv has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="cluster-manager-c5d6c587f-fd8rf", region="westus3", severity="warning"
  Description: Pod multicluster-engine/cluster-manager-c5d6c587f-fd8rf has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="multicluster-engine-operator-79c768d748-lg8vd", region="westus3", severity="warning"
  Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-lg8vd has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="kube-applier", pod="kube-applier-8475cd7c99-dfzv8", region="westus3", severity="warning"
  Description: Pod kube-applier/kube-applier-8475cd7c99-dfzv8 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="ocm-webhook-68676899fd-mklv5", region="westus3", severity="warning"
  Description: Pod multicluster-engine/ocm-webhook-68676899fd-mklv5 has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="mgmt-agent", pod="mgmt-agent-6dc8498b6f-ntd7x", region="westus3", severity="warning"
  Description: Pod mgmt-agent/mgmt-agent-6dc8498b6f-ntd7x has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="managedcluster-import-controller-v2-6cb46794f7-bzb25", region="westus3", severity="warning"
  Description: Pod multicluster-engine/managedcluster-import-controller-v2-6cb46794f7-bzb25 has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-27T15:12:43Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9877504-mgmt-2", component="kubernetes-infrastructure", namespace="secret-sync-controller", pod="secrets-store-sync-controller-manager-7f4b9bc5df-9hg5c", region="westus3", severity="warning"
  Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-9hg5c has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-l96rfk"
Unexpected error:
    <*fmt.wrapError | 0x11cfecaf8760>: 
    [strict TLS verification] route was never reachable: Get "https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        msg: "[strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
        err: <*url.Error | 0x11cfee5e70e0>{
            Op: "Get",
            URL: "https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net",
            Err: <*tls.CertificateVerificationError | 0x11cfee5e70b0>{
                UnverifiedCertificates: [
                    {
                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],
                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],
                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1],
                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],
                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],
                        Signature: [50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],
                        SignatureAlgorithm: 4,
                        PublicKeyAlgorithm: 1,
                        PublicKey: <*rsa.PublicKey | 0x11cfee823170>{
                            N: {
                                neg: false,
                                abs: [12328844732508617451, 3386053924852305530, 11508787352079574519, 580165219016783073, 15126087406076015328, 8473747687808762055, 6638534397921598209, 4531117369475629035, 8686734636532653775, 9699186360867822250, 15005114980278798855, 3900631334283717631, 15336487295176330797, 13597435546774256175, 1139063161098112232, 10161195232013165903, 15551291529941624294, 8053304939740217029, 18299432943387333464, 12035498982612883405, 1076299365375342957, 3325077180629581387, 4778720590180299818, 5926909629444315306, 16602979226450151451, 14838171221845856625, 18308053321550668292, 3951715190352458341, 11733219666321171503, 11793477282006375930, 17943060274730047763, 12841897204482256645],
                            },
                            E: 65537,
                        },
                        Version: 3,
                        SerialNumber: {
                            neg: false,
                            abs: [8845870848929380208],
                        },
                        Issuer: {
                            Country: nil,
                            Organization: nil,
                            OrganizationalUnit: ["openshift"],
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "root-ca",
                            Names: [
                                {
                                    Type: [2, 5, 4, 11],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"root-ca",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        Subject: {
                            Country: nil,
                            Organization: ["openshift"],
                            OrganizationalUnit: nil,
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "openshift-ingress",
                            Names: [
                                {
                                    Type: [2, 5, 4, 10],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"openshift-ingress",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        NotBefore: {wall: 0, ext: 63928477681, loc: nil},
                        NotAfter: {wall: 0, ext: 63960013681, loc: nil},
                        KeyUsage: 5,
                        Extensions: [
                            {
                                Id: [2, 5, 29, 15],
                                Critical: true,
                                Value: [3, 2, 5, 160],
                            },
                            {
                                Id: [2, 5, 29, 37],
                                Critical: false,
                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],
                            },
                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},
                            {
                                Id: [2, 5, 29, 14],
                                Critical: false,
                                Value: [4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],
                            },
                            {
                                Id: [2, 5, 29, 35],
                                Critical: false,
                                Value: [48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],
                            },
                            {
                                Id: [2, 5, 29, 17],
                                Critical: false,
                                Value: "0C\x82A*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net",
                            },
                        ],
                        ExtraExtensions: nil,
                        UnhandledCriticalExtensions: nil,
                        ExtKeyUsage: [2, 1],
                        UnknownExtKeyUsage: nil,
                        BasicConstraintsValid: true,
                        IsCA: false,
                        MaxPathLen: -1,
                        MaxPathLenZero: false,
                        SubjectKeyId: [197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],
                        AuthorityKeyId: [248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],
                        OCSPServer: nil,
                        IssuingCertificateURL: nil,
                        DNSNames: [
                            "*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net",
                        ],
                        EmailAddresses: nil,
                        IPAddresses: nil,
                        URIs: nil,
                        PermittedDNSDomainsCritical: false,
                        PermittedDNSDomains: nil,
                        ExcludedDNSDomains: nil,
                        PermittedIPRanges: nil,
                        ExcludedIPRanges: nil,
                        PermittedEmailAddresses: nil,
                        ExcludedEmailAddresses: nil,
                        PermittedURIDomains: nil,
                        ExcludedURIDomains: nil,
                        CRLDistributionPoints: nil,
                        PolicyIdentifiers: nil,
                        Policies: nil,
                        InhibitAnyPolicy: 0,
                        InhibitAnyPolicyZero: false,
                        InhibitPolicyMapping: 0,
                        InhibitPolicyMappingZero: false,
                        RequireExplicitPolicy: 0,
                        RequireExplicitPolicyZero: false,
                        PolicyMappings: nil,
                    },
                ],
                Err: <x509.UnknownAuthorityError>{
                    Cert: 0x11cfed040008 (cyclic reference),
                    hintErr: nil,
                    hintCert: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-l96rfk"
Unexpected error:
    <*fmt.wrapError | 0x11cfecaf8760>: 
    [strict TLS verification] route was never reachable: Get "https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        msg: "[strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
        err: <*url.Error | 0x11cfee5e70e0>{
            Op: "Get",
            URL: "https://agnhost-e2e-sample-app-4vg4t.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net",
            Err: <*tls.CertificateVerificationError | 0x11cfee5e70b0>{
                UnverifiedCertificates: [
                    {
                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],
                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 122, 194, 216, 171, 111, 107, 55, 112, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 50, 50, 56, 48, 49, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 111, 54, 101, 52, 109, 52, 117, 53, 117, 52, 116, 52, 99, 52, 117, 46, 115, 98, 112, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],
                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 178, 55, 149, 119, 219, 63, 47, 5, 249, 2, 142, 64, 226, 181, 25, 19, 163, 170, 217, 26, 124, 161, 29, 250, 162, 212, 197, 31, 10, 179, 100, 47, 54, 215, 80, 7, 78, 121, 50, 101, 254, 19, 69, 126, 231, 240, 34, 4, 205, 235, 198, 28, 58, 139, 209, 113, 230, 105, 161, 163, 221, 192, 72, 27, 82, 64, 156, 231, 230, 222, 216, 170, 66, 81, 109, 14, 218, 138, 180, 42, 46, 37, 12, 20, 42, 46, 254, 75, 14, 239, 200, 146, 1, 44, 249, 109, 167, 6, 174, 142, 209, 55, 87, 205, 253, 244, 165, 78, 181, 174, 211, 88, 111, 195, 22, 44, 169, 29, 62, 197, 215, 209, 73, 64, 144, 182, 37, 230, 141, 3, 209, 58, 41, 137, 165, 79, 15, 206, 195, 232, 245, 252, 140, 232, 188, 179, 203, 150, 101, 191, 186, 47, 212, 214, 37, 240, 83, 70, 206, 45, 54, 33, 211, 136, 55, 189, 43, 255, 208, 60, 224, 146, 245, 105, 10, 7, 134, 154, 110, 155, 81, 215, 26, 170, 120, 141, 123, 32, 172, 156, 94, 207, 62, 225, 195, 61, 155, 50, 127, 235, 92, 32, 207, 220, 201, 84, 27, 1, 117, 152, 204, 160, 80, 33, 0, 199, 209, 234, 168, 90, 184, 171, 178, 224, 8, 13, 41, 49, 94, 136, 52, 225, 159, 183, 109, 30, 220, 54, 37, 247, 46, 253, 174, 26, 197, 35, 142, 122, 171, 24, 218, 246, 148, 31, 58, 235, 2, 3, 1, 0, 1],
                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],
                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],
                        Signature: [50, 25, 231, 199, 177, 69, 58, 16, 51, 91, 243, 219, 210, 199, 138, 155, 53, 245, 245, 106, 127, 215, 113, 116, 53, 212, 199, 147, 106, 103, 247, 189, 33, 171, 252, 21, 143, 150, 56, 31, 85, 0, 41, 147, 65, 24, 201, 140, 203, 150, 3, 31, 110, 137, 35, 75, 171, 175, 235, 209, 21, 26, 164, 179, 71, 67, 21, 133, 105, 14, 253, 71, 51, 45, 32, 43, 100, 94, 84, 47, 64, 244, 56, 58, 136, 157, 46, 87, 211, 40, 31, 180, 2, 165, 196, 210, 222, 204, 152, 251, 130, 232, 9, 182, 120, 239, 235, 22, 89, 58, 198, 78, 82, 139, 120, 162, 101, 8, 223, 107, 59, 223, 193, 151, 252, 30, 79, 0, 15, 40, 112, 104, 59, 157, 68, 154, 197, 178, 218, 209, 164, 56, 94, 93, 22, 22, 98, 191, 126, 106, 4, 194, 233, 190, 68, 193, 123, 232, 86, 187, 65, 26, 135, 65, 91, 129, 105, 54, 92, 78, 247, 245, 172, 13, 38, 206, 186, 5, 136, 153, 211, 200, 140, 153, 205, 176, 93, 247, 111, 230, 227, 13, 56, 73, 251, 143, 236, 121, 113, 183, 124, 61, 169, 206, 40, 146, 37, 166, 252, 132, 4, 153, 168, 84, 39, 23, 232, 218, 136, 80, 210, 143, 97, 106, 58, 193, 56, 253, 19, 202, 243, 199, 49, 235, 83, 136, 105, 217, 57, 123, 238, 192, 105, 226, 211, 224, 255, 115, 130, 221, 177, 40, 19, 51, 31, 249],
                        SignatureAlgorithm: 4,
                        PublicKeyAlgorithm: 1,
                        PublicKey: <*rsa.PublicKey | 0x11cfee823170>{
                            N: {
                                neg: false,
                                abs: [12328844732508617451, 3386053924852305530, 11508787352079574519, 580165219016783073, 15126087406076015328, 8473747687808762055, 6638534397921598209, 4531117369475629035, 8686734636532653775, 9699186360867822250, 15005114980278798855, 3900631334283717631, 15336487295176330797, 13597435546774256175, 1139063161098112232, 10161195232013165903, 15551291529941624294, 8053304939740217029, 18299432943387333464, 12035498982612883405, 1076299365375342957, 3325077180629581387, 4778720590180299818, 5926909629444315306, 16602979226450151451, 14838171221845856625, 18308053321550668292, 3951715190352458341, 11733219666321171503, 11793477282006375930, 17943060274730047763, 12841897204482256645],
                            },
                            E: 65537,
                        },
                        Version: 3,
                        SerialNumber: {
                            neg: false,
                            abs: [8845870848929380208],
                        },
                        Issuer: {
                            Country: nil,
                            Organization: nil,
                            OrganizationalUnit: ["openshift"],
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "root-ca",
                            Names: [
                                {
                                    Type: [2, 5, 4, 11],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"root-ca",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        Subject: {
                            Country: nil,
                            Organization: ["openshift"],
                            OrganizationalUnit: nil,
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "openshift-ingress",
                            Names: [
                                {
                                    Type: [2, 5, 4, 10],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"openshift-ingress",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        NotBefore: {wall: 0, ext: 63928477681, loc: nil},
                        NotAfter: {wall: 0, ext: 63960013681, loc: nil},
                        KeyUsage: 5,
                        Extensions: [
                            {
                                Id: [2, 5, 29, 15],
                                Critical: true,
                                Value: [3, 2, 5, 160],
                            },
                            {
                                Id: [2, 5, 29, 37],
                                Critical: false,
                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],
                            },
                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},
                            {
                                Id: [2, 5, 29, 14],
                                Critical: false,
                                Value: [4, 64, 197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],
                            },
                            {
                                Id: [2, 5, 29, 35],
                                Critical: false,
                                Value: [48, 66, 128, 64, 248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],
                            },
                            {
                                Id: [2, 5, 29, 17],
                                Critical: false,
                                Value: "0C\x82A*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net",
                            },
                        ],
                        ExtraExtensions: nil,
                        UnhandledCriticalExtensions: nil,
                        ExtKeyUsage: [2, 1],
                        UnknownExtKeyUsage: nil,
                        BasicConstraintsValid: true,
                        IsCA: false,
                        MaxPathLen: -1,
                        MaxPathLenZero: false,
                        SubjectKeyId: [197, 98, 86, 250, 137, 217, 107, 201, 120, 11, 250, 28, 8, 159, 50, 58, 147, 2, 127, 27, 39, 46, 176, 228, 89, 113, 119, 156, 18, 254, 186, 225, 239, 236, 201, 198, 127, 141, 38, 73, 33, 245, 173, 2, 193, 175, 61, 245, 132, 223, 97, 52, 79, 55, 120, 201, 63, 76, 209, 109, 45, 189, 34, 122],
                        AuthorityKeyId: [248, 50, 11, 119, 242, 227, 194, 167, 149, 215, 190, 179, 176, 143, 74, 56, 221, 252, 111, 212, 224, 237, 204, 6, 0, 197, 97, 180, 119, 93, 170, 15, 222, 152, 153, 8, 168, 38, 140, 186, 14, 89, 130, 247, 23, 168, 143, 191, 156, 56, 163, 177, 216, 127, 102, 115, 15, 90, 73, 84, 30, 110, 35, 172],
                        OCSPServer: nil,
                        IssuingCertificateURL: nil,
                        DNSNames: [
                            "*.apps.aro.o6e4m4u5u4t4c4u.sbp8.westus3.aroapp-hcp.azure-test.net",
                        ],
                        EmailAddresses: nil,
                        IPAddresses: nil,
                        URIs: nil,
                        PermittedDNSDomainsCritical: false,
                        PermittedDNSDomains: nil,
                        ExcludedDNSDomains: nil,
                        PermittedIPRanges: nil,
                        ExcludedIPRanges: nil,
                        PermittedEmailAddresses: nil,
                        ExcludedEmailAddresses: nil,
                        PermittedURIDomains: nil,
                        ExcludedURIDomains: nil,
                        CRLDistributionPoints: nil,
                        PolicyIdentifiers: nil,
                        Policies: nil,
                        InhibitAnyPolicy: 0,
                        InhibitAnyPolicyZero: false,
                        InhibitPolicyMapping: 0,
                        InhibitPolicyMappingZero: false,
                        RequireExplicitPolicy: 0,
                        RequireExplicitPolicyZero: false,
                        PolicyMappings: nil,
                    },
                ],
                Err: <x509.UnknownAuthorityError>{
                    Cert: 0x11cfed040008 (cyclic reference),
                    hintErr: nil,
                    hintCert: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-2rcw86"
Unexpected error:
    <*fmt.wrapError | 0x2b2e578984a0>: 
    [strict TLS verification] route was never reachable: Get "https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        msg: "[strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
        err: <*url.Error | 0x2b2e57370780>{
            Op: "Get",
            URL: "https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net",
            Err: <*tls.CertificateVerificationError | 0x2b2e57370750>{
                UnverifiedCertificates: [
                    {
                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],
                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],
                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1],
                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],
                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],
                        Signature: [144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],
                        SignatureAlgorithm: 4,
                        PublicKeyAlgorithm: 1,
                        PublicKey: <*rsa.PublicKey | 0x2b2e57843d70>{
                            N: {
                                neg: false,
                                abs: [2877128800215149205, 16629919685921545055, 13732506504615110447, 11103748926260740575, 11416989105297106818, 9223988340828442177, 2897202458616478060, 11766261893188694937, 15086238765284949963, 10809216967110507236, 9979912210168109780, 11664503868611822661, 3024932991847888604, 17852206095763650445, 1698336212534998063, 17691123283498957122, 9751057521779199143, 5748498642685419890, 4821085446021960184, 2270194980282856385, 5998191060056422898, 1979344939107882427, 11002240530182040303, 17961926253159653156, 4444200366252054316, 17136494472856312874, 5046575580329891885, 18333653388545728381, 18332734810168546880, 2830450700003086872, 11572718067588258363, 13260460152132755496],
                            },
                            E: 65537,
                        },
                        Version: 3,
                        SerialNumber: {
                            neg: false,
                            abs: [8959707475658302546],
                        },
                        Issuer: {
                            Country: nil,
                            Organization: nil,
                            OrganizationalUnit: ["openshift"],
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "root-ca",
                            Names: [
                                {
                                    Type: [2, 5, 4, 11],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"root-ca",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        Subject: {
                            Country: nil,
                            Organization: ["openshift"],
                            OrganizationalUnit: nil,
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "openshift-ingress",
                            Names: [
                                {
                                    Type: [2, 5, 4, 10],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"openshift-ingress",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        NotBefore: {wall: 0, ext: 63928472888, loc: nil},
                        NotAfter: {wall: 0, ext: 63960008888, loc: nil},
                        KeyUsage: 5,
                        Extensions: [
                            {
                                Id: [2, 5, 29, 15],
                                Critical: true,
                                Value: [3, 2, 5, 160],
                            },
                            {
                                Id: [2, 5, 29, 37],
                                Critical: false,
                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],
                            },
                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},
                            {
                                Id: [2, 5, 29, 14],
                                Critical: false,
                                Value: [4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],
                            },
                            {
                                Id: [2, 5, 29, 35],
                                Critical: false,
                                Value: [48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],
                            },
                            {
                                Id: [2, 5, 29, 17],
                                Critical: false,
                                Value: "0C\x82A*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net",
                            },
                        ],
                        ExtraExtensions: nil,
                        UnhandledCriticalExtensions: nil,
                        ExtKeyUsage: [2, 1],
                        UnknownExtKeyUsage: nil,
                        BasicConstraintsValid: true,
                        IsCA: false,
                        MaxPathLen: -1,
                        MaxPathLenZero: false,
                        SubjectKeyId: [43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],
                        AuthorityKeyId: [145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],
                        OCSPServer: nil,
                        IssuingCertificateURL: nil,
                        DNSNames: [
                            "*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net",
                        ],
                        EmailAddresses: nil,
                        IPAddresses: nil,
                        URIs: nil,
                        PermittedDNSDomainsCritical: false,
                        PermittedDNSDomains: nil,
                        ExcludedDNSDomains: nil,
                        PermittedIPRanges: nil,
                        ExcludedIPRanges: nil,
                        PermittedEmailAddresses: nil,
                        ExcludedEmailAddresses: nil,
                        PermittedURIDomains: nil,
                        ExcludedURIDomains: nil,
                        CRLDistributionPoints: nil,
                        PolicyIdentifiers: nil,
                        Policies: nil,
                        InhibitAnyPolicy: 0,
                        InhibitAnyPolicyZero: false,
                        InhibitPolicyMapping: 0,
                        InhibitPolicyMappingZero: false,
                        RequireExplicitPolicy: 0,
                        RequireExplicitPolicyZero: false,
                        PolicyMappings: nil,
                    },
                ],
                Err: <x509.UnknownAuthorityError>{
                    Cert: 0x2b2e55ba2008 (cyclic reference),
                    hintErr: nil,
                    hintCert: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-2rcw86"
Unexpected error:
    <*fmt.wrapError | 0x2b2e578984a0>: 
    [strict TLS verification] route was never reachable: Get "https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        msg: "[strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
        err: <*url.Error | 0x2b2e57370780>{
            Op: "Get",
            URL: "https://agnhost-e2e-sample-app-w86zx.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net",
            Err: <*tls.CertificateVerificationError | 0x2b2e57370750>{
                UnverifiedCertificates: [
                    {
                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],
                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 124, 87, 70, 122, 130, 6, 100, 82, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 23, 13, 50, 55, 49, 48, 50, 52, 50, 49, 48, 56, 48, 56, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 97, 50, 115, 57, 98, 56, 109, 50, 101, 52, 112, 55, 112, 48, 103, 46, 48, 109, 117, 56, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],
                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 184, 6, 158, 64, 94, 24, 96, 40, 160, 154, 141, 196, 128, 87, 218, 59, 39, 71, 199, 234, 79, 240, 146, 24, 254, 106, 245, 45, 201, 136, 214, 64, 254, 110, 56, 158, 251, 103, 159, 125, 70, 9, 9, 192, 157, 185, 88, 45, 237, 209, 14, 238, 22, 11, 156, 42, 61, 172, 248, 175, 205, 29, 87, 44, 249, 69, 148, 194, 201, 88, 31, 36, 152, 175, 207, 120, 70, 231, 234, 239, 27, 120, 11, 188, 47, 126, 21, 187, 83, 61, 218, 253, 69, 150, 57, 242, 31, 129, 90, 78, 122, 162, 115, 193, 66, 231, 239, 171, 129, 241, 209, 248, 79, 198, 197, 15, 5, 20, 97, 114, 135, 82, 183, 39, 26, 12, 204, 167, 245, 131, 126, 229, 14, 152, 249, 66, 23, 145, 179, 201, 129, 84, 180, 47, 247, 191, 198, 219, 235, 14, 55, 141, 41, 250, 184, 135, 128, 115, 246, 220, 161, 224, 164, 119, 161, 199, 4, 69, 138, 127, 197, 71, 128, 167, 90, 212, 150, 2, 13, 143, 216, 191, 58, 228, 209, 93, 22, 58, 9, 137, 127, 203, 163, 74, 40, 217, 253, 186, 51, 153, 40, 52, 238, 73, 60, 57, 57, 108, 128, 2, 48, 134, 115, 108, 142, 65, 158, 113, 75, 25, 249, 189, 39, 130, 154, 24, 112, 209, 120, 229, 129, 223, 190, 147, 169, 232, 115, 83, 39, 47, 230, 201, 87, 216, 98, 161, 27, 95, 39, 237, 157, 102, 109, 222, 170, 149, 2, 3, 1, 0, 1],
                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],
                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],
                        Signature: [144, 63, 1, 9, 63, 203, 104, 158, 72, 190, 218, 173, 218, 174, 171, 175, 249, 99, 198, 163, 154, 183, 80, 70, 18, 90, 143, 16, 110, 40, 74, 15, 152, 104, 155, 119, 3, 189, 92, 182, 181, 193, 112, 193, 179, 171, 177, 195, 218, 45, 246, 105, 209, 209, 4, 147, 233, 109, 252, 165, 52, 200, 179, 47, 64, 202, 66, 155, 170, 48, 184, 165, 51, 160, 106, 229, 210, 44, 72, 109, 204, 32, 9, 38, 222, 130, 253, 176, 6, 246, 50, 104, 240, 45, 124, 77, 233, 84, 164, 109, 154, 110, 171, 77, 164, 253, 134, 225, 58, 131, 116, 48, 78, 1, 85, 150, 81, 149, 67, 171, 102, 216, 126, 229, 90, 56, 26, 33, 171, 124, 161, 209, 140, 187, 61, 34, 212, 215, 17, 134, 62, 6, 13, 208, 221, 55, 5, 187, 22, 63, 93, 164, 76, 124, 166, 239, 104, 210, 84, 78, 146, 114, 146, 113, 215, 97, 33, 13, 186, 135, 110, 182, 194, 44, 64, 215, 36, 43, 178, 21, 170, 171, 251, 180, 100, 40, 239, 189, 24, 174, 192, 195, 98, 108, 222, 96, 135, 172, 56, 114, 81, 213, 239, 84, 199, 27, 118, 56, 193, 239, 243, 191, 178, 97, 106, 201, 66, 252, 31, 190, 254, 107, 226, 184, 19, 188, 113, 44, 99, 24, 245, 52, 154, 97, 249, 178, 221, 232, 203, 63, 51, 4, 109, 167, 92, 246, 143, 188, 32, 32, 87, 98, 186, 5, 69, 17],
                        SignatureAlgorithm: 4,
                        PublicKeyAlgorithm: 1,
                        PublicKey: <*rsa.PublicKey | 0x2b2e57843d70>{
                            N: {
                                neg: false,
                                abs: [2877128800215149205, 16629919685921545055, 13732506504615110447, 11103748926260740575, 11416989105297106818, 9223988340828442177, 2897202458616478060, 11766261893188694937, 15086238765284949963, 10809216967110507236, 9979912210168109780, 11664503868611822661, 3024932991847888604, 17852206095763650445, 1698336212534998063, 17691123283498957122, 9751057521779199143, 5748498642685419890, 4821085446021960184, 2270194980282856385, 5998191060056422898, 1979344939107882427, 11002240530182040303, 17961926253159653156, 4444200366252054316, 17136494472856312874, 5046575580329891885, 18333653388545728381, 18332734810168546880, 2830450700003086872, 11572718067588258363, 13260460152132755496],
                            },
                            E: 65537,
                        },
                        Version: 3,
                        SerialNumber: {
                            neg: false,
                            abs: [8959707475658302546],
                        },
                        Issuer: {
                            Country: nil,
                            Organization: nil,
                            OrganizationalUnit: ["openshift"],
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "root-ca",
                            Names: [
                                {
                                    Type: [2, 5, 4, 11],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"root-ca",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        Subject: {
                            Country: nil,
                            Organization: ["openshift"],
                            OrganizationalUnit: nil,
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "openshift-ingress",
                            Names: [
                                {
                                    Type: [2, 5, 4, 10],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"openshift-ingress",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        NotBefore: {wall: 0, ext: 63928472888, loc: nil},
                        NotAfter: {wall: 0, ext: 63960008888, loc: nil},
                        KeyUsage: 5,
                        Extensions: [
                            {
                                Id: [2, 5, 29, 15],
                                Critical: true,
                                Value: [3, 2, 5, 160],
                            },
                            {
                                Id: [2, 5, 29, 37],
                                Critical: false,
                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],
                            },
                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},
                            {
                                Id: [2, 5, 29, 14],
                                Critical: false,
                                Value: [4, 64, 43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],
                            },
                            {
                                Id: [2, 5, 29, 35],
                                Critical: false,
                                Value: [48, 66, 128, 64, 145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],
                            },
                            {
                                Id: [2, 5, 29, 17],
                                Critical: false,
                                Value: "0C\x82A*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net",
                            },
                        ],
                        ExtraExtensions: nil,
                        UnhandledCriticalExtensions: nil,
                        ExtKeyUsage: [2, 1],
                        UnknownExtKeyUsage: nil,
                        BasicConstraintsValid: true,
                        IsCA: false,
                        MaxPathLen: -1,
                        MaxPathLenZero: false,
                        SubjectKeyId: [43, 61, 184, 58, 190, 246, 117, 145, 150, 149, 52, 148, 50, 70, 171, 24, 140, 56, 164, 203, 105, 63, 65, 146, 183, 66, 114, 129, 170, 239, 140, 79, 13, 223, 192, 116, 206, 195, 234, 70, 154, 102, 164, 7, 77, 72, 8, 99, 67, 120, 61, 224, 48, 163, 84, 4, 250, 254, 27, 88, 242, 13, 99, 21],
                        AuthorityKeyId: [145, 115, 72, 248, 147, 3, 210, 10, 72, 90, 51, 130, 36, 255, 101, 123, 45, 151, 78, 158, 148, 254, 229, 16, 121, 180, 193, 131, 183, 127, 36, 88, 142, 33, 22, 114, 228, 61, 202, 29, 164, 17, 211, 221, 107, 244, 245, 206, 36, 16, 177, 96, 20, 74, 22, 17, 184, 132, 100, 119, 214, 222, 88, 253],
                        OCSPServer: nil,
                        IssuingCertificateURL: nil,
                        DNSNames: [
                            "*.apps.aro.a2s9b8m2e4p7p0g.0mu8.westus3.aroapp-hcp.azure-test.net",
                        ],
                        EmailAddresses: nil,
                        IPAddresses: nil,
                        URIs: nil,
                        PermittedDNSDomainsCritical: false,
                        PermittedDNSDomains: nil,
                        ExcludedDNSDomains: nil,
                        PermittedIPRanges: nil,
                        ExcludedIPRanges: nil,
                        PermittedEmailAddresses: nil,
                        ExcludedEmailAddresses: nil,
                        PermittedURIDomains: nil,
                        ExcludedURIDomains: nil,
                        CRLDistributionPoints: nil,
                        PolicyIdentifiers: nil,
                        Policies: nil,
                        InhibitAnyPolicy: 0,
                        InhibitAnyPolicyZero: false,
                        InhibitPolicyMapping: 0,
                        InhibitPolicyMappingZero: false,
                        RequireExplicitPolicy: 0,
                        RequireExplicitPolicyZero: false,
                        PolicyMappings: nil,
                    },
                ],
                Err: <x509.UnknownAuthorityError>{
                    Cert: 0x2b2e55ba2008 (cyclic reference),
                    hintErr: nil,
                    hintCert: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-wk4gjj"
Unexpected error:
    <*fmt.wrapError | 0x30fb7a9c6d80>: 
    [strict TLS verification] route was never reachable: Get "https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        msg: "[strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
        err: <*url.Error | 0x30fb7cb9b320>{
            Op: "Get",
            URL: "https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net",
            Err: <*tls.CertificateVerificationError | 0x30fb7cb9b2f0>{
                UnverifiedCertificates: [
                    {
                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],
                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],
                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1],
                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],
                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],
                        Signature: [62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],
                        SignatureAlgorithm: 4,
                        PublicKeyAlgorithm: 1,
                        PublicKey: <*rsa.PublicKey | 0x30fb7a9895b0>{
                            N: {
                                neg: false,
                                abs: [3713275565900749005, 5533862720488119315, 886043176200789153, 17818264461995138607, 16265450768615453130, 5834472357489801107, 5862144673391444614, 628656117149906395, 13178964295004987606, 715618696217184398, 12066541168516629782, 6050057140458382287, 14400619811794133371, 6862502860246136883, 1422013248764323211, 8372522752467421288, 13138907997452357860, 17826693266249031145, 5098815638656127509, 5738821655630622400, 12438080729488309683, 6057798802862875801, 7503648299216627878, 13557164215630681915, 9649412887563066602, 17200267080417426931, 12116969928591911697, 17745395029646614848, 8889710084128733264, 16509317662742049607, 11507875977146301207, 12660817078547036528],
                            },
                            E: 65537,
                        },
                        Version: 3,
                        SerialNumber: {
                            neg: false,
                            abs: [7949786151822990365],
                        },
                        Issuer: {
                            Country: nil,
                            Organization: nil,
                            OrganizationalUnit: ["openshift"],
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "root-ca",
                            Names: [
                                {
                                    Type: [2, 5, 4, 11],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"root-ca",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        Subject: {
                            Country: nil,
                            Organization: ["openshift"],
                            OrganizationalUnit: nil,
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "openshift-ingress",
                            Names: [
                                {
                                    Type: [2, 5, 4, 10],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"openshift-ingress",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        NotBefore: {wall: 0, ext: 63928459200, loc: nil},
                        NotAfter: {wall: 0, ext: 63959995200, loc: nil},
                        KeyUsage: 5,
                        Extensions: [
                            {
                                Id: [2, 5, 29, 15],
                                Critical: true,
                                Value: [3, 2, 5, 160],
                            },
                            {
                                Id: [2, 5, 29, 37],
                                Critical: false,
                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],
                            },
                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},
                            {
                                Id: [2, 5, 29, 14],
                                Critical: false,
                                Value: [4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],
                            },
                            {
                                Id: [2, 5, 29, 35],
                                Critical: false,
                                Value: [48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],
                            },
                            {
                                Id: [2, 5, 29, 17],
                                Critical: false,
                                Value: "0C\x82A*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net",
                            },
                        ],
                        ExtraExtensions: nil,
                        UnhandledCriticalExtensions: nil,
                        ExtKeyUsage: [2, 1],
                        UnknownExtKeyUsage: nil,
                        BasicConstraintsValid: true,
                        IsCA: false,
                        MaxPathLen: -1,
                        MaxPathLenZero: false,
                        SubjectKeyId: [184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],
                        AuthorityKeyId: [107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],
                        OCSPServer: nil,
                        IssuingCertificateURL: nil,
                        DNSNames: [
                            "*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net",
                        ],
                        EmailAddresses: nil,
                        IPAddresses: nil,
                        URIs: nil,
                        PermittedDNSDomainsCritical: false,
                        PermittedDNSDomains: nil,
                        ExcludedDNSDomains: nil,
                        PermittedIPRanges: nil,
                        ExcludedIPRanges: nil,
                        PermittedEmailAddresses: nil,
                        ExcludedEmailAddresses: nil,
                        PermittedURIDomains: nil,
                        ExcludedURIDomains: nil,
                        CRLDistributionPoints: nil,
                        PolicyIdentifiers: nil,
                        Policies: nil,
                        InhibitAnyPolicy: 0,
                        InhibitAnyPolicyZero: false,
                        InhibitPolicyMapping: 0,
                        InhibitPolicyMappingZero: false,
                        RequireExplicitPolicy: 0,
                        RequireExplicitPolicyZero: false,
                        PolicyMappings: nil,
                    },
                ],
                Err: <x509.UnknownAuthorityError>{
                    Cert: 0x30fb7a8f2c08 (cyclic reference),
                    hintErr: nil,
                    hintCert: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-wk4gjj"
Unexpected error:
    <*fmt.wrapError | 0x30fb7a9c6d80>: 
    [strict TLS verification] route was never reachable: Get "https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        msg: "[strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
        err: <*url.Error | 0x30fb7cb9b320>{
            Op: "Get",
            URL: "https://agnhost-e2e-sample-app-25lgf.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net",
            Err: <*tls.CertificateVerificationError | 0x30fb7cb9b2f0>{
                UnverifiedCertificates: [
                    {
                        Raw: [48, 130, 4, 1, 48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 3, 130, 1, 1, 0, 62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],
                        RawTBSCertificate: [48, 130, 2, 233, 160, 3, 2, 1, 2, 2, 8, 110, 83, 80, 98, 163, 186, 156, 29, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 11, 5, 0, 48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97, 48, 30, 23, 13, 50, 54, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 23, 13, 50, 55, 49, 48, 50, 52, 49, 55, 50, 48, 48, 48, 90, 48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115, 48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1, 163, 130, 1, 39, 48, 130, 1, 35, 48, 14, 6, 3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 5, 160, 48, 29, 6, 3, 85, 29, 37, 4, 22, 48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1, 48, 12, 6, 3, 85, 29, 19, 1, 1, 255, 4, 2, 48, 0, 48, 73, 6, 3, 85, 29, 14, 4, 66, 4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128, 48, 75, 6, 3, 85, 29, 35, 4, 68, 48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41, 48, 76, 6, 3, 85, 29, 17, 4, 69, 48, 67, 130, 65, 42, 46, 97, 112, 112, 115, 46, 97, 114, 111, 46, 99, 57, 112, 48, 118, 55, 106, 57, 113, 55, 106, 52, 97, 49, 113, 46, 97, 102, 108, 100, 46, 119, 101, 115, 116, 117, 115, 51, 46, 97, 114, 111, 97, 112, 112, 45, 104, 99, 112, 46, 97, 122, 117, 114, 101, 45, 116, 101, 115, 116, 46, 110, 101, 116],
                        RawSubjectPublicKeyInfo: [48, 130, 1, 34, 48, 13, 6, 9, 42, 134, 72, 134, 247, 13, 1, 1, 1, 5, 0, 3, 130, 1, 15, 0, 48, 130, 1, 10, 2, 130, 1, 1, 0, 175, 180, 66, 13, 148, 254, 185, 112, 159, 180, 48, 58, 217, 14, 123, 23, 229, 28, 224, 241, 179, 210, 115, 71, 123, 94, 152, 56, 200, 179, 120, 80, 246, 68, 78, 194, 165, 17, 161, 64, 168, 40, 31, 243, 155, 207, 171, 17, 238, 179, 159, 199, 90, 32, 61, 243, 133, 233, 153, 229, 126, 69, 220, 234, 188, 36, 185, 6, 114, 77, 215, 59, 104, 34, 80, 95, 66, 69, 104, 166, 84, 17, 159, 234, 101, 109, 164, 153, 172, 156, 240, 134, 16, 200, 61, 179, 79, 164, 99, 227, 229, 46, 174, 192, 70, 194, 161, 207, 5, 78, 18, 21, 247, 101, 35, 19, 102, 159, 29, 233, 182, 86, 199, 53, 244, 175, 208, 228, 116, 49, 45, 21, 118, 178, 72, 104, 19, 188, 1, 131, 254, 127, 37, 139, 95, 60, 129, 254, 7, 18, 184, 51, 199, 217, 71, 107, 203, 113, 33, 123, 83, 246, 30, 234, 39, 214, 99, 207, 167, 116, 247, 66, 241, 15, 133, 22, 9, 238, 99, 105, 166, 167, 72, 142, 182, 229, 22, 51, 127, 84, 32, 214, 8, 185, 111, 104, 100, 152, 181, 219, 81, 90, 133, 133, 127, 114, 2, 134, 80, 248, 53, 178, 96, 3, 247, 147, 225, 186, 125, 75, 198, 237, 177, 202, 247, 71, 49, 31, 123, 47, 222, 47, 12, 75, 219, 147, 5, 27, 92, 161, 76, 204, 58, 211, 26, 120, 156, 19, 51, 136, 52, 116, 18, 166, 216, 205, 2, 3, 1, 0, 1],
                        RawSubject: [48, 48, 49, 18, 48, 16, 6, 3, 85, 4, 10, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 26, 48, 24, 6, 3, 85, 4, 3, 19, 17, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 105, 110, 103, 114, 101, 115, 115],
                        RawIssuer: [48, 38, 49, 18, 48, 16, 6, 3, 85, 4, 11, 19, 9, 111, 112, 101, 110, 115, 104, 105, 102, 116, 49, 16, 48, 14, 6, 3, 85, 4, 3, 19, 7, 114, 111, 111, 116, 45, 99, 97],
                        Signature: [62, 37, 28, 104, 244, 85, 128, 159, 33, 132, 56, 114, 149, 155, 76, 150, 56, 190, 182, 145, 23, 80, 230, 251, 69, 191, 95, 185, 206, 12, 203, 223, 159, 176, 108, 31, 228, 30, 29, 147, 188, 224, 159, 222, 209, 80, 26, 111, 48, 5, 218, 228, 214, 243, 4, 174, 170, 173, 102, 5, 47, 112, 16, 26, 249, 76, 134, 179, 187, 39, 158, 241, 110, 27, 4, 56, 253, 35, 167, 85, 21, 75, 203, 60, 52, 83, 13, 153, 94, 185, 69, 153, 196, 55, 251, 226, 149, 80, 157, 57, 244, 94, 195, 179, 137, 174, 166, 85, 12, 161, 202, 212, 188, 14, 213, 25, 59, 74, 197, 148, 98, 52, 206, 23, 41, 197, 32, 214, 165, 60, 202, 205, 67, 25, 42, 38, 206, 141, 201, 237, 101, 166, 209, 131, 101, 208, 225, 123, 244, 235, 82, 50, 143, 145, 70, 217, 191, 27, 3, 130, 194, 10, 227, 129, 5, 165, 150, 45, 143, 154, 59, 167, 145, 196, 78, 140, 76, 209, 96, 166, 23, 54, 227, 85, 65, 210, 246, 64, 194, 155, 249, 210, 255, 119, 224, 229, 103, 89, 68, 18, 164, 119, 9, 207, 73, 245, 219, 68, 214, 3, 155, 42, 94, 212, 192, 93, 148, 231, 125, 114, 77, 118, 234, 177, 58, 169, 209, 85, 145, 75, 243, 82, 234, 209, 51, 198, 131, 119, 63, 14, 40, 242, 219, 78, 215, 35, 237, 91, 234, 2, 142, 86, 121, 121, 249, 150],
                        SignatureAlgorithm: 4,
                        PublicKeyAlgorithm: 1,
                        PublicKey: <*rsa.PublicKey | 0x30fb7a9895b0>{
                            N: {
                                neg: false,
                                abs: [3713275565900749005, 5533862720488119315, 886043176200789153, 17818264461995138607, 16265450768615453130, 5834472357489801107, 5862144673391444614, 628656117149906395, 13178964295004987606, 715618696217184398, 12066541168516629782, 6050057140458382287, 14400619811794133371, 6862502860246136883, 1422013248764323211, 8372522752467421288, 13138907997452357860, 17826693266249031145, 5098815638656127509, 5738821655630622400, 12438080729488309683, 6057798802862875801, 7503648299216627878, 13557164215630681915, 9649412887563066602, 17200267080417426931, 12116969928591911697, 17745395029646614848, 8889710084128733264, 16509317662742049607, 11507875977146301207, 12660817078547036528],
                            },
                            E: 65537,
                        },
                        Version: 3,
                        SerialNumber: {
                            neg: false,
                            abs: [7949786151822990365],
                        },
                        Issuer: {
                            Country: nil,
                            Organization: nil,
                            OrganizationalUnit: ["openshift"],
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "root-ca",
                            Names: [
                                {
                                    Type: [2, 5, 4, 11],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"root-ca",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        Subject: {
                            Country: nil,
                            Organization: ["openshift"],
                            OrganizationalUnit: nil,
                            Locality: nil,
                            Province: nil,
                            StreetAddress: nil,
                            PostalCode: nil,
                            SerialNumber: "",
                            CommonName: "openshift-ingress",
                            Names: [
                                {
                                    Type: [2, 5, 4, 10],
                                    Value: <string>"openshift",
                                },
                                {
                                    Type: [2, 5, 4, 3],
                                    Value: <string>"openshift-ingress",
                                },
                            ],
                            ExtraNames: nil,
                        },
                        NotBefore: {wall: 0, ext: 63928459200, loc: nil},
                        NotAfter: {wall: 0, ext: 63959995200, loc: nil},
                        KeyUsage: 5,
                        Extensions: [
                            {
                                Id: [2, 5, 29, 15],
                                Critical: true,
                                Value: [3, 2, 5, 160],
                            },
                            {
                                Id: [2, 5, 29, 37],
                                Critical: false,
                                Value: [48, 20, 6, 8, 43, 6, 1, 5, 5, 7, 3, 2, 6, 8, 43, 6, 1, 5, 5, 7, 3, 1],
                            },
                            {Id: [2, 5, 29, 19], Critical: true, Value: [48, 0]},
                            {
                                Id: [2, 5, 29, 14],
                                Critical: false,
                                Value: [4, 64, 184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],
                            },
                            {
                                Id: [2, 5, 29, 35],
                                Critical: false,
                                Value: [48, 66, 128, 64, 107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],
                            },
                            {
                                Id: [2, 5, 29, 17],
                                Critical: false,
                                Value: "0C\x82A*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net",
                            },
                        ],
                        ExtraExtensions: nil,
                        UnhandledCriticalExtensions: nil,
                        ExtKeyUsage: [2, 1],
                        UnknownExtKeyUsage: nil,
                        BasicConstraintsValid: true,
                        IsCA: false,
                        MaxPathLen: -1,
                        MaxPathLenZero: false,
                        SubjectKeyId: [184, 27, 111, 100, 185, 3, 60, 74, 232, 82, 187, 169, 146, 147, 185, 95, 74, 1, 30, 137, 225, 41, 223, 249, 128, 141, 254, 150, 25, 225, 242, 3, 235, 207, 147, 83, 107, 148, 237, 0, 226, 240, 42, 175, 47, 113, 123, 148, 128, 190, 12, 146, 119, 216, 235, 223, 47, 29, 124, 171, 8, 133, 245, 128],
                        AuthorityKeyId: [107, 35, 93, 140, 120, 74, 68, 242, 166, 39, 191, 159, 156, 255, 152, 139, 116, 160, 21, 220, 61, 249, 246, 136, 64, 148, 57, 252, 107, 221, 102, 126, 167, 207, 36, 223, 46, 112, 92, 229, 162, 35, 131, 186, 43, 139, 91, 12, 221, 184, 234, 124, 29, 20, 52, 38, 121, 7, 144, 253, 45, 84, 166, 41],
                        OCSPServer: nil,
                        IssuingCertificateURL: nil,
                        DNSNames: [
                            "*.apps.aro.c9p0v7j9q7j4a1q.afld.westus3.aroapp-hcp.azure-test.net",
                        ],
                        EmailAddresses: nil,
                        IPAddresses: nil,
                        URIs: nil,
                        PermittedDNSDomainsCritical: false,
                        PermittedDNSDomains: nil,
                        ExcludedDNSDomains: nil,
                        PermittedIPRanges: nil,
                        ExcludedIPRanges: nil,
                        PermittedEmailAddresses: nil,
                        ExcludedEmailAddresses: nil,
                        PermittedURIDomains: nil,
                        ExcludedURIDomains: nil,
                        CRLDistributionPoints: nil,
                        PolicyIdentifiers: nil,
                        Policies: nil,
                        InhibitAnyPolicy: 0,
                        InhibitAnyPolicyZero: false,
                        InhibitPolicyMapping: 0,
                        InhibitPolicyMappingZero: false,
                        RequireExplicitPolicy: 0,
                        RequireExplicitPolicyZero: false,
                        PolicyMappings: nil,
                    },
                ],
                Err: <x509.UnknownAuthorityError>{
                    Cert: 0x30fb7a8f2c08 (cyclic reference),
                    hintErr: nil,
                    hintCert: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth
Unexpected error:
    <*fmt.wrapErrors | 0x1977dbbad170>: 
    failed waiting for external auth "external-auth" in resourcegroup="external-auth-cluster-c7xfsrnzff2g" for cluster="ea-cluster" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded
    {
        msg: "failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-c7xfsrnzff2g\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x1977dcaa2220>{
                s: "timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth
Unexpected error:
    <*fmt.wrapErrors | 0x1977dbbad170>: 
    failed waiting for external auth "external-auth" in resourcegroup="external-auth-cluster-c7xfsrnzff2g" for cluster="ea-cluster" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded
    {
        msg: "failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-c7xfsrnzff2g\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x1977dcaa2220>{
                s: "timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-c7xfsrnzff2g",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config "ext-auth-cidr"
Unexpected error:
    <*fmt.wrapErrors | 0x2cc075309380>: 
    failed waiting for external auth "ext-auth-cidr" in resourcegroup="e2e-cidr-connectivity-rhgbbsmxx2nr" for cluster="cidr-connectivity-test" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded
    {
        msg: "failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-rhgbbsmxx2nr\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x2cc074e74090>{
                s: "timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config "ext-auth-cidr"
Unexpected error:
    <*fmt.wrapErrors | 0x2cc075309380>: 
    failed waiting for external auth "ext-auth-cidr" in resourcegroup="e2e-cidr-connectivity-rhgbbsmxx2nr" for cluster="cidr-connectivity-test" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded
    {
        msg: "failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-rhgbbsmxx2nr\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x2cc074e74090>{
                s: "timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-rhgbbsmxx2nr",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth
Unexpected error:
    <*fmt.wrapErrors | 0x32cbbfa08480>: 
    failed waiting for external auth "external-auth" in resourcegroup="external-auth-cluster-s8rx2m7kc8lm" for cluster="ea-cluster" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded
    {
        msg: "failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-s8rx2m7kc8lm\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x32cbbf682f10>{
                s: "timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:181]: failed to create external auth config external-auth
Unexpected error:
    <*fmt.wrapErrors | 0x32cbbfa08480>: 
    failed waiting for external auth "external-auth" in resourcegroup="external-auth-cluster-s8rx2m7kc8lm" for cluster="ea-cluster" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded
    {
        msg: "failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-s8rx2m7kc8lm\" for cluster=\"ea-cluster\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x32cbbf682f10>{
                s: "timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-s8rx2m7kc8lm",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0xdad4da81e80>: 
    DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0xdad4d9b41a0>{
            msg: "DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0xdad4da81e80>: 
    DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0xdad4d9b41a0>{
            msg: "DNS for agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-4gnm9.apps.aro.v20261001.bhnx.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-5-1-5f9zxt"
Unexpected error:
    <*fmt.wrapError | 0x3db0320ca900>: 
    DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x3db031a2bd20>{
            msg: "DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster "cluster-candidate-5-1-5f9zxt"
Unexpected error:
    <*fmt.wrapError | 0x3db0320ca900>: 
    DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x3db031a2bd20>{
            msg: "DNS for agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8tqcz.apps.aro.x0a9b9z6x4r0r3x.a2p1.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x1342e25441a0>: 
    DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x1342e251c060>{
            msg: "DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x1342e25441a0>: 
    DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x1342e251c060>{
            msg: "DNS for agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6g5k9.apps.aro.v20261001.1f89.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x24681019eca0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-m6dlx86v7zvt" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9",
      "name": "4249c9a2-6a00-4a31-a13c-7b395eeda2a9",
      "status": "Failed",
      "startTime": "2026-10-27T21:21:19.945219062Z",
      "endTime": "2026-10-27T21:50:23.791391981Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9
            correlation ID: 79f0a27c-81eb-45e3-ba1c-7ed22276e9b1
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:150]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x24681019eca0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-m6dlx86v7zvt" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9",
      "name": "4249c9a2-6a00-4a31-a13c-7b395eeda2a9",
      "status": "Failed",
      "startTime": "2026-10-27T21:21:19.945219062Z",
      "endTime": "2026-10-27T21:50:23.791391981Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4249c9a2-6a00-4a31-a13c-7b395eeda2a9
            correlation ID: 79f0a27c-81eb-45e3-ba1c-7ed22276e9b1
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-x79lxq7pcp2s/cluster-candidate-4-20-wz9qjx should provision
Unexpected error:
    <*fmt.wrapError | 0x1ea33da4c3e0>: 
    failed to create HCP cluster cluster-candidate-4-20-wz9qjx: failed waiting for cluster="cluster-candidate-4-20-wz9qjx" in resourcegroup="rg-candidate-4-20-x79lxq7pcp2s" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44",
      "name": "5ef510b2-7fbf-47d3-a252-f868567b1a44",
      "status": "Failed",
      "startTime": "2026-10-27T16:50:03.68579052Z",
      "endTime": "2026-10-27T17:19:03.174752422Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44
            correlation ID: 0794d196-09b5-4818-a7ca-7f5fb13bca4d
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:154]: HCP cluster rg-candidate-4-20-x79lxq7pcp2s/cluster-candidate-4-20-wz9qjx should provision
Unexpected error:
    <*fmt.wrapError | 0x1ea33da4c3e0>: 
    failed to create HCP cluster cluster-candidate-4-20-wz9qjx: failed waiting for cluster="cluster-candidate-4-20-wz9qjx" in resourcegroup="rg-candidate-4-20-x79lxq7pcp2s" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44",
      "name": "5ef510b2-7fbf-47d3-a252-f868567b1a44",
      "status": "Failed",
      "startTime": "2026-10-27T16:50:03.68579052Z",
      "endTime": "2026-10-27T17:19:03.174752422Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5ef510b2-7fbf-47d3-a252-f868567b1a44
            correlation ID: 0794d196-09b5-4818-a7ca-7f5fb13bca4d
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-22-njchjb" with version 4.22.18 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x2294f905a340>: 
    failed to create HCP cluster cluster-zstream-4-22-njchjb: failed waiting for cluster="cluster-zstream-4-22-njchjb" in resourcegroup="rg-zstream-upgrade-4-22-5bksdkff6tvq" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a",
      "name": "b7f42e30-e443-47e9-982e-3fe8abc06d9a",
      "status": "Failed",
      "startTime": "2026-10-24T22:19:20.094796575Z",
      "endTime": "2026-10-24T22:48:29.265123983Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a
            correlation ID: c6c76fd1-9a00-45d4-a377-4b7ed8ffb192
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-22-njchjb" with version 4.22.18 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x2294f905a340>: 
    failed to create HCP cluster cluster-zstream-4-22-njchjb: failed waiting for cluster="cluster-zstream-4-22-njchjb" in resourcegroup="rg-zstream-upgrade-4-22-5bksdkff6tvq" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a",
      "name": "b7f42e30-e443-47e9-982e-3fe8abc06d9a",
      "status": "Failed",
      "startTime": "2026-10-24T22:19:20.094796575Z",
      "endTime": "2026-10-24T22:48:29.265123983Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f42e30-e443-47e9-982e-3fe8abc06d9a
            correlation ID: c6c76fd1-9a00-45d4-a377-4b7ed8ffb192
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI
Unexpected error:
    <*errors.joinError | 0x248e56e12588>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x248e56f14360>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x248e5757aa00>{
                    msg: "DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x248e57018e60>{
                        msg: "DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI
Unexpected error:
    <*errors.joinError | 0x248e56e12588>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x248e56f14360>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x248e5757aa00>{
                    msg: "DNS for route host agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x248e57018e60>{
                        msg: "DNS for agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-84lh2.apps.aro.cilium-cl.t19t.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update
Unexpected error:
    <*errors.joinError | 0x22d4bdfbc060>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x22d4bdf9c1a0>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x22d4bdee0f00>{
                    msg: "DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x22d4bdf9da20>{
                        msg: "DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update
Unexpected error:
    <*errors.joinError | 0x22d4bdfbc060>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x22d4bdf9c1a0>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x22d4bdee0f00>{
                    msg: "DNS for route host agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x22d4bdf9da20>{
                        msg: "DNS for agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mcq7p.apps.aro.m0y7f5l6s2i7t2m.fbfu.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update
Unexpected error:
    <*errors.joinError | 0x1b9fbf4865b8>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x1b9fbd955860>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x1b9fbd298140>{
                    msg: "DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x1b9fbe030640>{
                        msg: "DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update
Unexpected error:
    <*errors.joinError | 0x1b9fbf4865b8>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x1b9fbd955860>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x1b9fbd298140>{
                    msg: "DNS for route host agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x1b9fbe030640>{
                        msg: "DNS for agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6gx6w.apps.aro.v0o5x2p3z8t0f8t.72eg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config
Unexpected error:
    <*fmt.wrapErrors | 0x13a2362ee210>: 
    failed waiting for hcpCluster="acr-pull" in resourcegroup="acr-pull-5cjwmbnpxtmc" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded
    {
        msg: "failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-5cjwmbnpxtmc\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x13a235e7d850>{
                s: "timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config
Unexpected error:
    <*fmt.wrapErrors | 0x13a2362ee210>: 
    failed waiting for hcpCluster="acr-pull" in resourcegroup="acr-pull-5cjwmbnpxtmc" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded
    {
        msg: "failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-5cjwmbnpxtmc\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x13a235e7d850>{
                s: "timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-5cjwmbnpxtmc",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config
Unexpected error:
    <*fmt.wrapErrors | 0xa204c386ab0>: 
    failed waiting for hcpCluster="acr-pull" in resourcegroup="acr-pull-zhz5sxs2r4gt" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded
    {
        msg: "failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-zhz5sxs2r4gt\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xa204b336550>{
                s: "timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:657]: failed to get admin REST config
Unexpected error:
    <*fmt.wrapErrors | 0xa204c386ab0>: 
    failed waiting for hcpCluster="acr-pull" in resourcegroup="acr-pull-zhz5sxs2r4gt" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded
    {
        msg: "failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-zhz5sxs2r4gt\" to finish getting creds, caused by: timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xa204b336550>{
                s: "timeout '2.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster20261001 for cluster acr-pull in resource group acr-pull-zhz5sxs2r4gt",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:118]: failed to create default nodepool np-1
Unexpected error:
    <*fmt.wrapError | 0x3cdcfbca5680>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "cluster-gpu-np" in resourcegroup="rg-gpu-nodepool-8vtwfvfssx9k" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0",
      "name": "907ac069-8701-4021-b410-cf10181345c0",
      "status": "Failed",
      "startTime": "2026-10-25T05:14:31.336281353Z",
      "endTime": "2026-10-25T05:33:33.578871475Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-p7hpl to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: Failed: interfaces failed to create or update. err: failed to get existing resource rg-gpu-nodepool-8vtwfvfssx9k--managed/cluster-gpu-np-np-1-8v7xk-p7hpl-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Network/networkInterfaces/cluster-gpu-np-np-1-8v7xk-p7hpl-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to ... message truncated\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0
            correlation ID: 1c529a86-2bdd-43d7-ab0e-4267cd216305
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:118]: failed to create default nodepool np-1
Unexpected error:
    <*fmt.wrapError | 0x3cdcfbca5680>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "cluster-gpu-np" in resourcegroup="rg-gpu-nodepool-8vtwfvfssx9k" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0",
      "name": "907ac069-8701-4021-b410-cf10181345c0",
      "status": "Failed",
      "startTime": "2026-10-25T05:14:31.336281353Z",
      "endTime": "2026-10-25T05:33:33.578871475Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-p7hpl to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cluster-gpu-np-np-1-8v7xk-p7hpl: Failed: interfaces failed to create or update. err: failed to get existing resource rg-gpu-nodepool-8vtwfvfssx9k--managed/cluster-gpu-np-np-1-8v7xk-p7hpl-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Network/networkInterfaces/cluster-gpu-np-np-1-8v7xk-p7hpl-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to ... message truncated\nMachine cluster-gpu-np-np-1-8v7xk-22mlx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-8vtwfvfssx9k--managed/providers/Microsoft.Compute/virtualMachines/cluster-gpu-np-np-1-8v7xk-22mlx to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/907ac069-8701-4021-b410-cf10181345c0
            correlation ID: 1c529a86-2bdd-43d7-ab0e-4267cd216305
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x3f57e0004820>: 
    DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x3f57dfc6e000>{
            msg: "DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x3f57e0004820>: 
    DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x3f57dfc6e000>{
            msg: "DNS for agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g99mf.apps.aro.v20260901.9ggg.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain
Expected success, but got an error:
    <*fmt.wrapError | 0x7f30d39c620>: 
    DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x7f30d39c600>{
            msg: "DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain
Expected success, but got an error:
    <*fmt.wrapError | 0x7f30d39c620>: 
    DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve: DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x7f30d39c600>{
            msg: "DNS for agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gt2zw.apps.aro.ingress-cert-le.msqo.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool "np-deny-pods" for cluster "cp-ystream-deny-pods-5-1-fjj5cf"
Unexpected error:
    <*fmt.wrapError | 0x3b484fe8ea00>: 
    failed to create NodePool np-deny-pods: failed waiting for nodepool="np-deny-pods" for cluster "cp-ystream-deny-pods-5-1-fjj5cf" in resourcegroup="rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a",
      "name": "cf1ae438-bf01-4f86-8ab5-784122b7cb5a",
      "status": "Failed",
      "startTime": "2026-10-24T03:36:53.315230765Z",
      "endTime": "2026-10-24T03:56:02.496643822Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j: Failed: interfaces failed to create or update. err: failed to get existing resource rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/providers/Microsoft.Network/networkInterfaces/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Ple... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a
            correlation ID: 3b16cd17-d78f-4b89-a83b-76ef2e36cf89
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool "np-deny-pods" for cluster "cp-ystream-deny-pods-5-1-fjj5cf"
Unexpected error:
    <*fmt.wrapError | 0x3b484fe8ea00>: 
    failed to create NodePool np-deny-pods: failed waiting for nodepool="np-deny-pods" for cluster "cp-ystream-deny-pods-5-1-fjj5cf" in resourcegroup="rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a",
      "name": "cf1ae438-bf01-4f86-8ab5-784122b7cb5a",
      "status": "Failed",
      "startTime": "2026-10-24T03:36:53.315230765Z",
      "endTime": "2026-10-24T03:56:02.496643822Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [hypershiftNodePool] node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j: Failed: interfaces failed to create or update. err: failed to get existing resource rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-1-j5b76vjznvvh-cp-ystream-den-2ba76655/providers/Microsoft.Network/networkInterfaces/z4z5b9p8d5g2j9j-np-deny-pods-8gxvs-vs65j-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '6096f642-63a0-4dd6-958d-715a87f33535' exceeded. Ple... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf1ae438-bf01-4f86-8ab5-784122b7cb5a
            correlation ID: 3b16cd17-d78f-4b89-a83b-76ef2e36cf89
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster "agg-cluster" with aggregated settings
Unexpected error:
    <*fmt.wrapError | 0x9134b719420>: 
    failed waiting for cluster="agg-cluster" in resourcegroup="feature-aggregation-c2zb7x4tvbjz" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f",
      "name": "2d4481c2-5c8d-4e2e-a526-9b051ca6139f",
      "status": "Failed",
      "startTime": "2026-10-24T17:16:34.851278228Z",
      "endTime": "2026-10-24T17:45:35.221810696Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f
            correlation ID: 4e6d7a8e-ea3e-4773-abae-818d5133da86
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster "agg-cluster" with aggregated settings
Unexpected error:
    <*fmt.wrapError | 0x9134b719420>: 
    failed waiting for cluster="agg-cluster" in resourcegroup="feature-aggregation-c2zb7x4tvbjz" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f",
      "name": "2d4481c2-5c8d-4e2e-a526-9b051ca6139f",
      "status": "Failed",
      "startTime": "2026-10-24T17:16:34.851278228Z",
      "endTime": "2026-10-24T17:45:35.221810696Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 22m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2d4481c2-5c8d-4e2e-a526-9b051ca6139f
            correlation ID: 4e6d7a8e-ea3e-4773-abae-818d5133da86
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1
Unexpected error:
    <*fmt.wrapError | 0x1e6e8804d3c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "negative-tests-cluster" in resourcegroup="rg-negative-tests-m5vrcvff2ctf" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8",
      "name": "75b8b1f9-f718-45b2-b1ee-a3009f824cf8",
      "status": "Failed",
      "startTime": "2026-10-22T14:53:42.149827787Z",
      "endTime": "2026-10-22T15:12:50.02702561Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8
            correlation ID: e990e88e-110c-474c-9878-aa84f73234da
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1
Unexpected error:
    <*fmt.wrapError | 0x1e6e8804d3c0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "negative-tests-cluster" in resourcegroup="rg-negative-tests-m5vrcvff2ctf" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8",
      "name": "75b8b1f9-f718-45b2-b1ee-a3009f824cf8",
      "status": "Failed",
      "startTime": "2026-10-22T14:53:42.149827787Z",
      "endTime": "2026-10-22T15:12:50.02702561Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-5kdm2 to exist\nMachine c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-m5vrcvff2ctf--managed/providers/Microsoft.Compute/virtualMachines/c2g2u9m5x1z4p0m-np-1-bfdzc-kbcws to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75b8b1f9-f718-45b2-b1ee-a3009f824cf8
            correlation ID: e990e88e-110c-474c-9878-aa84f73234da
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_etcd_mhsm.go:106]: failed to deploy and activate Managed HSM
Unexpected error:
    <*fmt.wrapError | 0x563a226e760>: 
    failed waiting for deployment "mhsm-deploy-etcd-mhsm" in resourcegroup="etcd-mhsm-blgzqwztp2gh" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeploymentFailed
    --------------------------------------------------------------------------------
    {
      "status": "Failed",
      "error": {
        "code": "DeploymentFailed",
        "message": "At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.",
        "details": [
          {
            "code": "Conflict",
            "message": "{\r\n  \"status\": \"failed\",\r\n  \"error\": {\r\n    \"code\": \"ResourceDeploymentFailure\",\r\n    \"message\": \"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\",\r\n    \"details\": [\r\n      {\r\n        \"code\": \"DeploymentScriptACIProvisioningTimeout\",\r\n        \"message\": \"The supporting resource: 'Azure Container Instance' did not provision during the expected time. Correlation Id: c1661afd-ea85-487b-8e90-65a3f8664855. Please try again later, if the issue persists contact technical support for further investigation.\"\r\n      }\r\n    ]\r\n  }\r\n}"
          }
        ]
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271
            correlation ID: 989b1e9c-002b-494d-ae3b-825cc5f4adf2
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_etcd_mhsm.go:106]: failed to deploy and activate Managed HSM
Unexpected error:
    <*fmt.wrapError | 0x563a226e760>: 
    failed waiting for deployment "mhsm-deploy-etcd-mhsm" in resourcegroup="etcd-mhsm-blgzqwztp2gh" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeploymentFailed
    --------------------------------------------------------------------------------
    {
      "status": "Failed",
      "error": {
        "code": "DeploymentFailed",
        "message": "At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.",
        "details": [
          {
            "code": "Conflict",
            "message": "{\r\n  \"status\": \"failed\",\r\n  \"error\": {\r\n    \"code\": \"ResourceDeploymentFailure\",\r\n    \"message\": \"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\",\r\n    \"details\": [\r\n      {\r\n        \"code\": \"DeploymentScriptACIProvisioningTimeout\",\r\n        \"message\": \"The supporting resource: 'Azure Container Instance' did not provision during the expected time. Correlation Id: c1661afd-ea85-487b-8e90-65a3f8664855. Please try again later, if the issue persists contact technical support for further investigation.\"\r\n      }\r\n    ]\r\n  }\r\n}"
          }
        ]
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/etcd-mhsm-blgzqwztp2gh/providers/Microsoft.Resources/deployments/mhsm-deploy-etcd-mhsm/operationStatuses/08584089270308456271
            correlation ID: 989b1e9c-002b-494d-ae3b-825cc5f4adf2
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster "cluster-candidate-5-1-g8n765"
Unexpected error:
    <*errors.joinError | 0x254aca410498>: 
    VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.008s: DNS A record "*.e2e-dns-c2hldx" not yet present in zone rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-c2hldx' does not exist in resource group 'rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx
            correlation ID: 192cc89e-15d5-4e23-8532-45b3cbb07059
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster "cluster-candidate-5-1-g8n765"
Unexpected error:
    <*errors.joinError | 0x254aca410498>: 
    VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.008s: DNS A record "*.e2e-dns-c2hldx" not yet present in zone rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-c2hldx' does not exist in resource group 'rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jqlpsk27qjbp-candidate-g8n765--managed/providers/Microsoft.Network/dnsZones/aro.u3d0c7m0f4m7m7j.sg1c.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-c2hldx
            correlation ID: 192cc89e-15d5-4e23-8532-45b3cbb07059
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:248]: failed to verify cluster operators on cluster "cluster-candidate-4-21-rf46z8"
Unexpected error:
    <*errors.joinError | 0x3ac741d4138>: 
    VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.001s: DNS A record "*.e2e-dns-fn44p6" not yet present in zone rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-fn44p6' does not exist in resource group 'rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6
            correlation ID: bf6f75cd-f285-4895-ad70-7d62eff4cba2
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:248]: failed to verify cluster operators on cluster "cluster-candidate-4-21-rf46z8"
Unexpected error:
    <*errors.joinError | 0x3ac741d4138>: 
    VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.001s: DNS A record "*.e2e-dns-fn44p6" not yet present in zone rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-fn44p6' does not exist in resource group 'rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vgzkq7npmrnr-candidate-rf46z8--managed/providers/Microsoft.Network/dnsZones/aro.l5z2t8a8n2n6x2i.7cr2.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-fn44p6
            correlation ID: bf6f75cd-f285-4895-ad70-7d62eff4cba2
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster "cluster-candidate-4-22-652kdm"
Unexpected error:
    <*errors.joinError | 0x282c0f144570>: 
    VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.006s: DNS A record "*.e2e-dns-jgd8x5" not yet present in zone rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-jgd8x5' does not exist in resource group 'rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5
            correlation ID: a77d314a-799e-4560-8840-6738610cf0af
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster "cluster-candidate-4-22-652kdm"
Unexpected error:
    <*errors.joinError | 0x282c0f144570>: 
    VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.006s: DNS A record "*.e2e-dns-jgd8x5" not yet present in zone rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-jgd8x5' does not exist in resource group 'rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-b89xsggl4sd5-candidate-652kdm--managed/providers/Microsoft.Network/dnsZones/aro.u8o6a8v1u0z8q8o.bssq.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-jgd8x5
            correlation ID: a77d314a-799e-4560-8840-6738610cf0af
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster "cluster-candidate-4-22-4fhvgp"
Unexpected error:
    <*errors.joinError | 0x3188fb29e5a0>: 
    VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.005s: DNS A record "*.e2e-dns-rg99mn" not yet present in zone rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-rg99mn' does not exist in resource group 'rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn
            correlation ID: cfd4563d-f15e-486d-ad09-c0f336987b6a
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:246]: failed to verify cluster operators on cluster "cluster-candidate-4-22-4fhvgp"
Unexpected error:
    <*errors.joinError | 0x3188fb29e5a0>: 
    VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) failed: VerifyIngressDNSRecord(aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net) timed out after 10m0.005s: DNS A record "*.e2e-dns-rg99mn" not yet present in zone rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: NotFound
    --------------------------------------------------------------------------------
    {
      "code": "NotFound",
      "message": "The resource record '*.e2e-dns-rg99mn' does not exist in resource group 'rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed' of subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'."
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sp7fr6hfrw9x-candidate-4fhvgp--managed/providers/Microsoft.Network/dnsZones/aro.d8c1z3j0o2p5g9d.3jyi.westus3.aroapp-hcp.azure-test.net/A/*.e2e-dns-rg99mn
            correlation ID: cfd4563d-f15e-486d-ad09-c0f336987b6a
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.
ClusterIssuer CRD should be installed and the resource creatable
Expected success, but got an error:
    <*errors.StatusError | 0x11d151799220>: 
    Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s": no endpoints available for service "cert-manager-webhook"
    {
        ErrStatus: {
            TypeMeta: {Kind: "Status", APIVersion: "v1"},
            ListMeta: {
                SelfLink: "",
                ResourceVersion: "",
                Continue: "",
                RemainingItemCount: nil,
                ShardInfo: nil,
            },
            Status: "Failure",
            Message: "Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
            Reason: "InternalError",
            Details: {
                Name: "",
                Group: "",
                Kind: "",
                UID: "",
                Causes: [
                    {
                        Type: "",
                        Message: "failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
                        Field: "",
                    },
                ],
                RetryAfterSeconds: 0,
            },
            Code: 500,
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.
ClusterIssuer CRD should be installed and the resource creatable
Expected success, but got an error:
    <*errors.StatusError | 0x11d151799220>: 
    Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s": no endpoints available for service "cert-manager-webhook"
    {
        ErrStatus: {
            TypeMeta: {Kind: "Status", APIVersion: "v1"},
            ListMeta: {
                SelfLink: "",
                ResourceVersion: "",
                Continue: "",
                RemainingItemCount: nil,
                ShardInfo: nil,
            },
            Status: "Failure",
            Message: "Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
            Reason: "InternalError",
            Details: {
                Name: "",
                Group: "",
                Kind: "",
                UID: "",
                Causes: [
                    {
                        Type: "",
                        Message: "failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"",
                        Field: "",
                    },
                ],
                RetryAfterSeconds: 0,
            },
            Code: 500,
        },
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-22-cghxqb"
Unexpected error:
    <*errors.joinError | 0x315939ea25b8>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv": nodes "p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0x31593941d8c0>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0x31593941d880>{
                    msg: "failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0x31593941d840>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0x31593a26f680>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:289]: failed to verify cluster operators on cluster "cluster-candidate-4-22-cghxqb"
Unexpected error:
    <*errors.joinError | 0x315939ea25b8>: 
    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node "p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv": nodes "p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized
    {
        errs: [
            <*fmt.wrapError | 0x31593941d8c0>{
                msg: "VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                err: <*fmt.wrapError | 0x31593941d880>{
                    msg: "failed to label node p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                    err: <*fmt.wrapError | 0x31593941d840>{
                        msg: "failed to apply k8s.ovn.org/egress-assignable label to node \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\": nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                        err: <*errors.StatusError | 0x31593a26f680>{
                            ErrStatus: {
                                TypeMeta: {Kind: "", APIVersion: ""},
                                ListMeta: {
                                    SelfLink: "",
                                    ResourceVersion: "",
                                    Continue: "",
                                    RemainingItemCount: nil,
                                    ShardInfo: nil,
                                },
                                Status: "Failure",
                                Message: "nodes \"p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized",
                                Reason: "Forbidden",
                                Details: {
                                    Name: "p3m2y8l9j1y3v1d-np-cghxqb-7rwcj-2g4cv",
                                    Group: "",
                                    Kind: "nodes",
                                    UID: "",
                                    Causes: nil,
                                    RetryAfterSeconds: 0,
                                },
                                Code: 403,
                            },
                        },
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.002s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x1e28491f1160>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416->172.30.0.10:53: read: connection refused",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.002s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x1e28491f1160>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.21:39416->172.30.0.10:53: read: connection refused",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster "cluster-candidate-4-20-4x46vw"
Unexpected error:
    <*fmt.wrapError | 0x3b1760358220>: 
    console fixture namespace "console-lrpv2qd": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-lrpv2qd"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition="ContainersReady" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="Initialized" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="PodScheduled" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="Ready" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp container="dex" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-lrpv2qd\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-lrpv2qd\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x3b1760b23ba0>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster "cluster-candidate-4-20-4x46vw"
Unexpected error:
    <*fmt.wrapError | 0x3b1760358220>: 
    console fixture namespace "console-lrpv2qd": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-lrpv2qd"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition="ContainersReady" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="Initialized" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="PodScheduled" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp condition="Ready" status="True" reason=""; Pod/dex-78f4dcc67d-zhbqp container="dex" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-lrpv2qd\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-lrpv2qd\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-zhbqp condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-zhbqp container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-zhbqp phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x3b1760b23ba0>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster "cluster-candidate-5-0-6rxg4b"
Unexpected error:
    <*fmt.wrapError | 0x12de498cec20>: 
    console fixture namespace "console-mr8rvbh": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-mr8rvbh"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition="ContainersReady" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="Initialized" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="PodScheduled" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="Ready" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j container="dex" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-mr8rvbh\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-mr8rvbh\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x12de4a14f120>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster "cluster-candidate-5-0-6rxg4b"
Unexpected error:
    <*fmt.wrapError | 0x12de498cec20>: 
    console fixture namespace "console-mr8rvbh": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-mr8rvbh"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition="ContainersReady" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="Initialized" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="PodScheduled" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j condition="Ready" status="True" reason=""; Pod/dex-77657bcfc5-zhr7j container="dex" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-mr8rvbh\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-mr8rvbh\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-zhr7j condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-zhr7j container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-zhr7j phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x12de4a14f120>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster "cluster-candidate-5-1-qx8d44"
Unexpected error:
    <*fmt.wrapError | 0x39e6141ab040>: 
    console fixture namespace "console-n2bkhw2": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-n2bkhw2"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition="ContainersReady" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="Initialized" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="PodScheduled" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="Ready" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d container="dex" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-n2bkhw2\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-n2bkhw2\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x39e614e48260>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster "cluster-candidate-5-1-qx8d44"
Unexpected error:
    <*fmt.wrapError | 0x39e6141ab040>: 
    console fixture namespace "console-n2bkhw2": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-n2bkhw2"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition="ContainersReady" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="Initialized" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="PodScheduled" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d condition="Ready" status="True" reason=""; Pod/dex-77657bcfc5-nrx7d container="dex" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-n2bkhw2\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-n2bkhw2\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-nrx7d condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-nrx7d container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-nrx7d phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x39e614e48260>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster "cluster-candidate-5-0-glc2bs"
Unexpected error:
    <*fmt.wrapError | 0x2d9cfabb2a80>: 
    console fixture namespace "console-r7gptrj": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-r7gptrj"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition="ContainersReady" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="Initialized" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="PodScheduled" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="Ready" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 container="dex" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-r7gptrj\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-r7gptrj\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x2d9cfac24920>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:197]: failed to set up Console login fixture for cluster "cluster-candidate-5-0-glc2bs"
Unexpected error:
    <*fmt.wrapError | 0x2d9cfabb2a80>: 
    console fixture namespace "console-r7gptrj": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-r7gptrj"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition="ContainersReady" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="Initialized" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="PodScheduled" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 condition="Ready" status="True" reason=""; Pod/dex-77657bcfc5-mwmw6 container="dex" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-r7gptrj\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-r7gptrj\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-77657bcfc5-mwmw6 condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-77657bcfc5-mwmw6 container=\"dex\" ready=true restarts=0; Pod/dex-77657bcfc5-mwmw6 phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x2d9cfac24920>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster "cluster-candidate-4-20-8dwfln"
Unexpected error:
    <*fmt.wrapError | 0x22821099b6a0>: 
    console fixture namespace "console-xzp4sq4": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-xzp4sq4"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition="ContainersReady" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="Initialized" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="PodScheduled" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="Ready" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr container="dex" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-xzp4sq4\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-xzp4sq4\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x228210e20900>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster "cluster-candidate-4-20-8dwfln"
Unexpected error:
    <*fmt.wrapError | 0x22821099b6a0>: 
    console fixture namespace "console-xzp4sq4": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-xzp4sq4"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition="ContainersReady" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="Initialized" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="PodScheduled" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr condition="Ready" status="True" reason=""; Pod/dex-78f4dcc67d-swzhr container="dex" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-xzp4sq4\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-xzp4sq4\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-swzhr condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-swzhr container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-swzhr phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x228210e20900>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster "cluster-candidate-4-20-9js6f6"
Unexpected error:
    <*fmt.wrapError | 0x10354e044ea0>: 
    console fixture namespace "console-z2wqbfw": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-z2wqbfw"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition="ContainersReady" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="Initialized" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="PodScheduled" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="Ready" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx container="dex" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-z2wqbfw\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-z2wqbfw\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x10354ed78880>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to set up Console login fixture for cluster "cluster-candidate-4-20-9js6f6"
Unexpected error:
    <*fmt.wrapError | 0x10354e044ea0>: 
    console fixture namespace "console-z2wqbfw": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace="console-z2wqbfw"; Deployment/dex condition="Available" status="True" reason="MinimumReplicasAvailable"; Deployment/dex condition="Progressing" status="True" reason="NewReplicaSetAvailable"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition="ContainersReady" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="Initialized" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="PodReadyToStartContainers" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="PodScheduled" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx condition="Ready" status="True" reason=""; Pod/dex-78f4dcc67d-6mptx container="dex" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase="Running" reason=""
    {
        msg: "console fixture namespace \"console-z2wqbfw\": configure Console ExternalAuth through RP: context deadline exceeded; Dex namespace=\"console-z2wqbfw\"; Deployment/dex condition=\"Available\" status=\"True\" reason=\"MinimumReplicasAvailable\"; Deployment/dex condition=\"Progressing\" status=\"True\" reason=\"NewReplicaSetAvailable\"; Deployment/dex generation=1 observed=1 replicas=1 ready=1 available=1; Dex Pods=1; Dex ReplicaSets=1; Pod/dex-78f4dcc67d-6mptx condition=\"ContainersReady\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Initialized\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodReadyToStartContainers\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"PodScheduled\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx condition=\"Ready\" status=\"True\" reason=\"\"; Pod/dex-78f4dcc67d-6mptx container=\"dex\" ready=true restarts=0; Pod/dex-78f4dcc67d-6mptx phase=\"Running\" reason=\"\"",
        err: <*fmt.wrapError | 0x10354ed78880>{
            msg: "configure Console ExternalAuth through RP: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-23-lkx8t5"
Expected success, but got an error:
    <*errors.errorString | 0x3306da82b390>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster "cluster-candidate-4-23-lkx8t5"
Expected success, but got an error:
    <*errors.errorString | 0x3306da82b390>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0xdf906826d40>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0xdf906826d40>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x3454a17e2bc0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret
Expected success, but got an error:
    <*errors.errorString | 0x3454a17e2bc0>: 
    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed
    {
        s: "ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-23-s7k2sd" at install version 4.22
Unexpected error:
    <*fmt.wrapError | 0x297062e2e820>: 
    failed to create HCP cluster cp-ystream-upgrade-4-23-s7k2sd: failed waiting for cluster="cp-ystream-upgrade-4-23-s7k2sd" in resourcegroup="rg-cp-ystream-upgrade-4-23-np7xjvwc8xct" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460",
      "name": "5d9c960a-8f4b-476d-bd5b-132c0ce76460",
      "status": "Failed",
      "startTime": "2026-10-28T20:12:32.825742556Z",
      "endTime": "2026-10-28T20:41:35.067951058Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460
            correlation ID: 00646159-dae9-4d36-b0f6-38a33eb4750d
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-23-s7k2sd" at install version 4.22
Unexpected error:
    <*fmt.wrapError | 0x297062e2e820>: 
    failed to create HCP cluster cp-ystream-upgrade-4-23-s7k2sd: failed waiting for cluster="cp-ystream-upgrade-4-23-s7k2sd" in resourcegroup="rg-cp-ystream-upgrade-4-23-np7xjvwc8xct" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460",
      "name": "5d9c960a-8f4b-476d-bd5b-132c0ce76460",
      "status": "Failed",
      "startTime": "2026-10-28T20:12:32.825742556Z",
      "endTime": "2026-10-28T20:41:35.067951058Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d9c960a-8f4b-476d-bd5b-132c0ce76460
            correlation ID: 00646159-dae9-4d36-b0f6-38a33eb4750d
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster
Unexpected error:
    <*fmt.wrapError | 0x10164f31ca00>: 
    failed to create HCP cluster basic-cluster: failed waiting for cluster="basic-cluster" in resourcegroup="rg-cluster-nsg-subnet-reuse-p669mb2thnqj" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b",
      "name": "f26dc7e7-f380-4b37-87cb-db720fa2be9b",
      "status": "Failed",
      "startTime": "2026-10-28T19:27:58.648367476Z",
      "endTime": "2026-10-28T19:57:03.816857754Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b
            correlation ID: b3870728-f2f7-48de-a0fa-7f5442586990
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster
Unexpected error:
    <*fmt.wrapError | 0x10164f31ca00>: 
    failed to create HCP cluster basic-cluster: failed waiting for cluster="basic-cluster" in resourcegroup="rg-cluster-nsg-subnet-reuse-p669mb2thnqj" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b",
      "name": "f26dc7e7-f380-4b37-87cb-db720fa2be9b",
      "status": "Failed",
      "startTime": "2026-10-28T19:27:58.648367476Z",
      "endTime": "2026-10-28T19:57:03.816857754Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f26dc7e7-f380-4b37-87cb-db720fa2be9b
            correlation ID: b3870728-f2f7-48de-a0fa-7f5442586990
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-21-lrsdfl" at install version 4.20
Unexpected error:
    <*fmt.wrapError | 0x3c350f225e0>: 
    failed to create HCP cluster cp-ystream-upgrade-4-21-lrsdfl: failed waiting for cluster="cp-ystream-upgrade-4-21-lrsdfl" in resourcegroup="rg-cp-ystream-upgrade-4-21-vfbxfhgg5xwh" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616",
      "name": "034eed87-a977-47cf-8ff5-2b8292aa4616",
      "status": "Failed",
      "startTime": "2026-10-28T20:12:34.208698264Z",
      "endTime": "2026-10-28T20:41:35.072616907Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616
            correlation ID: 880b9f59-ff57-4ef3-9ec1-2561f0663b62
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster "cp-ystream-upgrade-4-21-lrsdfl" at install version 4.20
Unexpected error:
    <*fmt.wrapError | 0x3c350f225e0>: 
    failed to create HCP cluster cp-ystream-upgrade-4-21-lrsdfl: failed waiting for cluster="cp-ystream-upgrade-4-21-lrsdfl" in resourcegroup="rg-cp-ystream-upgrade-4-21-vfbxfhgg5xwh" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616",
      "name": "034eed87-a977-47cf-8ff5-2b8292aa4616",
      "status": "Failed",
      "startTime": "2026-10-28T20:12:34.208698264Z",
      "endTime": "2026-10-28T20:41:35.072616907Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/034eed87-a977-47cf-8ff5-2b8292aa4616
            correlation ID: 880b9f59-ff57-4ef3-9ec1-2561f0663b62
            client request ID: unavailable
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable
Unexpected error:
    <*errors.joinError | 0x5c10dd80030>: 
    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized
    VerifyBasicAccess failed: failed to list services: Unauthorized
    {
        errs: [
            <*fmt.wrapError | 0x5c10d35ddc0>{
                msg: "VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized",
                err: <*fmt.wrapError | 0x5c10d35dda0>{
                    msg: "failed to list all APIServices: Unauthorized",
                    err: <*errors.StatusError | 0x5c10db35540>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
            <*fmt.wrapError | 0x5c10dd88020>{
                msg: "VerifyBasicAccess failed: failed to list services: Unauthorized",
                err: <*fmt.wrapError | 0x5c10dd88000>{
                    msg: "failed to list services: Unauthorized",
                    err: <*errors.StatusError | 0x5c10dbae320>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:254]: cluster oidc-wi-cluster is not viable
Unexpected error:
    <*errors.joinError | 0x5c10dd80030>: 
    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized
    VerifyBasicAccess failed: failed to list services: Unauthorized
    {
        errs: [
            <*fmt.wrapError | 0x5c10d35ddc0>{
                msg: "VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized",
                err: <*fmt.wrapError | 0x5c10d35dda0>{
                    msg: "failed to list all APIServices: Unauthorized",
                    err: <*errors.StatusError | 0x5c10db35540>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
            <*fmt.wrapError | 0x5c10dd88020>{
                msg: "VerifyBasicAccess failed: failed to list services: Unauthorized",
                err: <*fmt.wrapError | 0x5c10dd88000>{
                    msg: "failed to list services: Unauthorized",
                    err: <*errors.StatusError | 0x5c10dbae320>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster "cluster-nightly-4-22-67pgbb" is viable
Unexpected error:
    <*errors.joinError | 0x21cecb2413f8>: 
    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized
    VerifyBasicAccess failed: failed to list services: Unauthorized
    {
        errs: [
            <*fmt.wrapError | 0x21cecb85e9a0>{
                msg: "VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized",
                err: <*fmt.wrapError | 0x21cecb85e980>{
                    msg: "failed to list all APIServices: Unauthorized",
                    err: <*errors.StatusError | 0x21cecae4e500>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
            <*fmt.wrapError | 0x21cecb471000>{
                msg: "VerifyBasicAccess failed: failed to list services: Unauthorized",
                err: <*fmt.wrapError | 0x21cecb470fe0>{
                    msg: "failed to list services: Unauthorized",
                    err: <*errors.StatusError | 0x21cecb1806e0>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster "cluster-nightly-4-22-67pgbb" is viable
Unexpected error:
    <*errors.joinError | 0x21cecb2413f8>: 
    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized
    VerifyBasicAccess failed: failed to list services: Unauthorized
    {
        errs: [
            <*fmt.wrapError | 0x21cecb85e9a0>{
                msg: "VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized",
                err: <*fmt.wrapError | 0x21cecb85e980>{
                    msg: "failed to list all APIServices: Unauthorized",
                    err: <*errors.StatusError | 0x21cecae4e500>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
            <*fmt.wrapError | 0x21cecb471000>{
                msg: "VerifyBasicAccess failed: failed to list services: Unauthorized",
                err: <*fmt.wrapError | 0x21cecb470fe0>{
                    msg: "failed to list services: Unauthorized",
                    err: <*errors.StatusError | 0x21cecb1806e0>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable
Unexpected error:
    <*errors.joinError | 0x13bc429c0870>: 
    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized
    VerifyBasicAccess failed: failed to list services: Unauthorized
    {
        errs: [
            <*fmt.wrapError | 0x13bc42aa9920>{
                msg: "VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized",
                err: <*fmt.wrapError | 0x13bc42aa9900>{
                    msg: "failed to list all APIServices: Unauthorized",
                    err: <*errors.StatusError | 0x13bc41a5d400>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
            <*fmt.wrapError | 0x13bc425c16c0>{
                msg: "VerifyBasicAccess failed: failed to list services: Unauthorized",
                err: <*fmt.wrapError | 0x13bc425c16a0>{
                    msg: "failed to list services: Unauthorized",
                    err: <*errors.StatusError | 0x13bc416d8960>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable
Unexpected error:
    <*errors.joinError | 0x13bc429c0870>: 
    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized
    VerifyBasicAccess failed: failed to list services: Unauthorized
    {
        errs: [
            <*fmt.wrapError | 0x13bc42aa9920>{
                msg: "VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized",
                err: <*fmt.wrapError | 0x13bc42aa9900>{
                    msg: "failed to list all APIServices: Unauthorized",
                    err: <*errors.StatusError | 0x13bc41a5d400>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
            <*fmt.wrapError | 0x13bc425c16c0>{
                msg: "VerifyBasicAccess failed: failed to list services: Unauthorized",
                err: <*fmt.wrapError | 0x13bc425c16a0>{
                    msg: "failed to list services: Unauthorized",
                    err: <*errors.StatusError | 0x13bc416d8960>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                                ShardInfo: nil,
                            },
                            Status: "Failure",
                            Message: "Unauthorized",
                            Reason: "Unauthorized",
                            Details: nil,
                            Code: 401,
                        },
                    },
                },
            },
        ],
    }
occurred