CIHealth
<
Single Day: Oct 14▾
Pick a day
>
View JSON API
↗ Open Failure patterns for this day

Environment: DEV

Time (UTC)RuntimeJobRegionFailed atResultPRFailed testsDetails
2h 10m 49s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 20ab25f
 westus3 e2e failed ★#7544 (open) 2
e2e (2)
cluster <resource-group>/<cluster> entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
23:59:39 UTC · failed at e2e · test ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-44v5fj7gn2vn/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-44v5fj7gn2vn/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
There is no failure as the matcher passed to Eventually succeeded on its most recent iteration
23:59:39 UTC · failed at e2e · test Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.003s.
delegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance
There is no failure as the matcher passed to Eventually succeeded on its most recent iteration
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.003s.
delegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance
There is no failure as the matcher passed to Eventually succeeded on its most recent iteration
2h 16m 56s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 1ecfbed · merge 5dea2e6
 centralus e2e failed ★#7532 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
23:32:03 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xb36d0212940>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T00:30:08.2263967Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T00:30:08.2263967Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0xb36d0212920>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T00:30:08.2263967Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0xb36cf27e550>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0xb36d0212900>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T00:30:08.2263967Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0xb36d137e150>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xb36d0212940>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-15T00:30:08.2263967Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T00:30:08.2263967Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0xb36d0212920>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T00:30:08.2263967Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0xb36cf27e550>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0xb36d0212900>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dp2n7z9g2mfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-15T00:30:08.2263967Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0xb36d137e150>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 1m 22s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 8091a26 · merge 8091a26
 centralus alert, e2e failed #7540 (merged) 2
e2e (1)
cluster <resource-group>/<cluster> entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
23:09:45 UTC · failed at e2e · test ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-tp82mqtbfnjs/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-tp82mqtbfnjs/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
alert (1)
alert [svc] LeaderElectionLeaseStale fired
23:09:45 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-15T00:52:07Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j2275200-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.169:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2275200/providers/microsoft.monitor/accounts/services-j2275200", namespace="klusterlet-2tesd0o3f60ell7kn84k7na2ifcblhbl", pod="arohcp-monitor-kube-state-metrics-7988dc648c-wclqz", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tesd0o3f60ell7kn84k7na2ifcblhbl on cluster ci01-j2275200-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
2h 13m 47s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head c8f4b45
 westus3 e2e failed ⚠#7416 (open) 1
e2e (1)
waiting for a presented DNS-01 challenge for *
23:01:28 UTC · failed at e2e · test Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.001s.
delegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued
Expected success, but got an error:
    <*errors.errorString | 0x2cc845a2b820>: 
    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud
    {
        s: "waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:444]: Timed out after 90.001s.
delegated NS and ACME TXT should resolve consistently on all authoritative nameservers, or the certificate should already be issued
Expected success, but got an error:
    <*errors.errorString | 0x2cc845a2b820>: 
    waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud
    {
        s: "waiting for a presented DNS-01 challenge for *.apps.aro.ingress-cert-le.p7kd.j2563328.hcp.osadev.cloud",
    }
2h 15m 28s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 53c3391 · merge b69b4de
 westus3 alert, e2e failed ⚠#7451 (merged) 2
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row
23:00:10 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1f63e25da900>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T00:18:45.8286809Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f", ActivityId:"3e4bf140-f035-443b-96b4-bed0d927cd4f", SubActivityId:"3e4bf140-f035-443b-96b4-bed0d927cd4f", ActivityType:"GW.Http.CallContext", ParentActivityId:"3e4bf140-f035-443b-96b4-bed0d927cd4f", ActivityStack:"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x1f63e28703a0>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x1f63e2d64800>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-15T00:18:45.8286809Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f",
                        ActivityId: "3e4bf140-f035-443b-96b4-bed0d927cd4f",
                        SubActivityId: "3e4bf140-f035-443b-96b4-bed0d927cd4f",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "3e4bf140-f035-443b-96b4-bed0d927cd4f",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1f63e25da900>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-15T00:18:45.8286809Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f", ActivityId:"3e4bf140-f035-443b-96b4-bed0d927cd4f", SubActivityId:"3e4bf140-f035-443b-96b4-bed0d927cd4f", ActivityType:"GW.Http.CallContext", ParentActivityId:"3e4bf140-f035-443b-96b4-bed0d927cd4f", ActivityStack:"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6k948d9nsvr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x1f63e28703a0>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-15T00:18:45.8286809Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f\", ActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", SubActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"3e4bf140-f035-443b-96b4-bed0d927cd4f\", ActivityStack:\"(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x1f63e2d64800>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-15T00:18:45.8286809Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f",
                        ActivityId: "3e4bf140-f035-443b-96b4-bed0d927cd4f",
                        SubActivityId: "3e4bf140-f035-443b-96b4-bed0d927cd4f",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "3e4bf140-f035-443b-96b4-bed0d927cd4f",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;e95fc3a4-a2a2-47ad-b9ce-87006b7b438f ARID=3e4bf140-f035-443b-96b4-bed0d927cd4f > GW.Http.CallContext/3e4bf140-f035-443b-96b4-bed0d927cd4f)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
23:00:10 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T00:13:57Z
  Ended:       2026-10-15T00:19:58Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j5903232-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 19m 36s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 4d44077
 westus3 alert failed #7526 (closed) 1
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
22:53:42 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 3 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-15T00:02:33Z
  Ended:       2026-10-15T00:17:34Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j8037120-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-15T00:02:33Z
  Ended:       2026-10-15T00:19:35Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j8037120-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 3:
  State:       Resolved
  Started:     2026-10-15T00:03:34Z
  Ended:       2026-10-15T00:15:37Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j8037120-svc", component="backend", name="clusterclusterserviceupdatedispatch", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 13m 19s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 574f74c
 centralus alert, e2e failed ★#6688 (open) 2
e2e (1)
| take 101, Post "<url>": context deadline exceeded
22:40:21 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1cdf5a79e560>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T23:20:39.7053116Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:20:39.7053116Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1cdf5a79e520>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:20:39.7053116Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1cdf5a5c4500>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1cdf5a79e500>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:20:39.7053116Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1cdf5bbab080>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1cdf5a79e560>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T23:20:39.7053116Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:20:39.7053116Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1cdf5a79e520>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:20:39.7053116Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1cdf5a5c4500>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1cdf5a79e500>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gmq4jh246r5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:20:39.7053116Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1cdf5bbab080>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
22:40:21 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T23:38:40Z
  Ended:       2026-10-14T23:56:38Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j0374272-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 18m 7s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head b81b319 · merge b81b319
 westus3 e2e failed ★#7342 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
22:39:40 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1b5b518622a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T23:28:59.0626387Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:28:59.0626387Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1b5b51862280>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:28:59.0626387Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1b5b50bb8140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1b5b51862260>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:28:59.0626387Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1b5b528f0810>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1b5b518622a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T23:28:59.0626387Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:28:59.0626387Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1b5b51862280>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:28:59.0626387Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1b5b50bb8140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1b5b51862260>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-spmr2bkfrwww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T23:28:59.0626387Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1b5b528f0810>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 11m 18s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head cf7a1e9 · merge cf7a1e9
 centralus n/a passed #7529 (merged) 0 n/a
2h 24m 24s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head c171e40 · merge c171e40
 centralus n/a passed #7519 (merged) 0 n/a
2h 9m 6s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 1d23c83
 centralus alert, e2e failed #6364 (open) 4
e2e (1)
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift
22:15:16 UTC · failed at e2e · test Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster "basic-hcp-cluster" to complete creation (timeout '30.000000' minutes)
Unexpected error:
    <*exported.ResponseError | 0x24524eb9ea50>: 
    GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b",
      "name": "812fe66b-c9fc-46f9-8d46-bf8a0cd5068b",
      "status": "Failed",
      "startTime": "2026-10-14T23:04:33.551295422Z",
      "endTime": "2026-10-14T23:33:41.530060174Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b
            correlation ID: d14d5694-03de-4ac8-b587-4d0e35762d84
            client request ID: 5cfd8941-3a60-4ebe-8bc6-a659901f1339
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster "basic-hcp-cluster" to complete creation (timeout '30.000000' minutes)
Unexpected error:
    <*exported.ResponseError | 0x24524eb9ea50>: 
    GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b",
      "name": "812fe66b-c9fc-46f9-8d46-bf8a0cd5068b",
      "status": "Failed",
      "startTime": "2026-10-14T23:04:33.551295422Z",
      "endTime": "2026-10-14T23:33:41.530060174Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0214528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/812fe66b-c9fc-46f9-8d46-bf8a0cd5068b
            correlation ID: d14d5694-03de-4ac8-b587-4d0e35762d84
            client request ID: 5cfd8941-3a60-4ebe-8bc6-a659901f1339
occurred
alert (3)
alert [hcp] KubeStatefulSetReplicasMismatch fired
22:15:16 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T23:34:33Z
  Ended:       2026-10-14T23:51:31Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j0214528-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.65.21:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528", namespace="ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u", pod="arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd has not matched the expected number of replicas for longer than 15 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-14T23:35:30Z
  Ended:       2026-10-14T23:52:28Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j0214528-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.65.21:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528", namespace="ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u", pod="arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd has not matched the expected number of replicas for longer than 15 minutes.
alert [hcp] KubeDeploymentReplicasMismatch fired
22:15:16 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T23:39:28Z
  Ended:       2026-10-14T23:53:28Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0214528-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.65.21:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528", namespace="ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u", pod="arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-14T23:39:28Z
  Ended:       2026-10-14T23:53:29Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0214528-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.65.21:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528", namespace="ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u", pod="arohcp-monitor-kube-state-metrics-7988dc648c-r2hbm", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
alert [hcp] KubePodCrashLooping fired
22:15:16 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T23:26:27Z
  Ended:       2026-10-14T23:56:29Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0214528-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.65.21:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528", namespace="ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="e7a6870f-14b4-45e0-9c19-fce7f03813dc"
  Description: Pod ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-14T23:26:27Z
  Ended:       2026-10-14T23:56:29Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0214528-mgmt-2", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.65.21:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0214528/providers/microsoft.monitor/accounts/hcps-j0214528", namespace="ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="e7a6870f-14b4-45e0-9c19-fce7f03813dc"
  Description: Pod ocm-arohcpci01-2terp08u81ovror5h3o9svngoo0a5t0t-q6f6k8k9x1s1l6u/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
2h 28m 9s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 5bf7c6e · merge 5bf7c6e
 westus3 e2e failed #7492 (merged) 1
e2e (1)
cluster <resource-group>/<cluster> entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
21:58:50 UTC · failed at e2e · test ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-2bxwcdlh6wnp/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-2bxwcdlh6wnp/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
2h 8m 53s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 8c3c716 · merge 8c3c716
 centralus n/a passed #7383 (merged) 0 n/a
2h 46m 25s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 413c69d · merge 09587f6
 centralus e2e failed ★#7440 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
21:30:23 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x38253b9f65a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T22:50:51.9885278Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T22:50:51.9885278Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x38253b9f6580>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T22:50:51.9885278Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3825394d9180>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x38253b9f6560>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T22:50:51.9885278Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x38253b805a10>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x38253b9f65a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T22:50:51.9885278Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T22:50:51.9885278Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x38253b9f6580>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T22:50:51.9885278Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3825394d9180>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x38253b9f6560>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-82dnc5wf2n2c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T22:50:51.9885278Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x38253b805a10>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 25m 54s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 8af0205 · merge 8af0205
 westus3 alert failed #7528 (merged) 1
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
21:30:15 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T22:50:55Z
  Ended:       2026-10-14T23:11:00Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j0486016-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 33m 11s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head b6c0794 · merge b6c0794
 westus3 alert, e2e failed ★#7524 (merged) 3
e2e (2)
failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T23:11:15
21:30:12 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 611.066s.
Expected success, but got an error:
    <*fmt.wrapError | 0x32cad0a314a0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);
    let _endTime = datetime(2026-10-14T23:11:15.1642696Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-lznjhs5kd54t'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T23:11:29.6178254Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0", ActivityId:"e2be55f9-a0e2-4a71-a19a-22ceddc025d8", SubActivityId:"e2be55f9-a0e2-4a71-a19a-22ceddc025d8", ActivityType:"GW.Http.CallContext", ParentActivityId:"e2be55f9-a0e2-4a71-a19a-22ceddc025d8", ActivityStack:"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x32cad17d37b8>{
            errs: [
                <*fmt.wrapError | 0x32cad0a31480>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x32cad0a31460>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x32cad0a313a0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x32cad0a31360>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x32cacfbb4000>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 611.066s.
Expected success, but got an error:
    <*fmt.wrapError | 0x32cad0a314a0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);
    let _endTime = datetime(2026-10-14T23:11:15.1642696Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-lznjhs5kd54t'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T23:11:29.6178254Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0", ActivityId:"e2be55f9-a0e2-4a71-a19a-22ceddc025d8", SubActivityId:"e2be55f9-a0e2-4a71-a19a-22ceddc025d8", ActivityType:"GW.Http.CallContext", ParentActivityId:"e2be55f9-a0e2-4a71-a19a-22ceddc025d8", ActivityStack:"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x32cad17d37b8>{
            errs: [
                <*fmt.wrapError | 0x32cad0a31480>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x32cad0a31460>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x32cad0a313a0>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x32cad0a31360>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T23:11:15.1642694Z);\nlet _endTime = datetime(2026-10-14T23:11:15.1642696Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-lznjhs5kd54t'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T23:11:29.6178254Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0\", ActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", SubActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"e2be55f9-a0e2-4a71-a19a-22ceddc025d8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5c9a447a-4db8-4265-ba5b-cc44d975cad0 ARID=e2be55f9-a0e2-4a71-a19a-22ceddc025d8 > GW.Http.CallContext/e2be55f9-a0e2-4a71-a19a-22ceddc025d8)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x32cacfbb4000>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "<url>
21:30:12 UTC · failed at e2e · test Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.007s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x3db417569280>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452->172.30.0.10:53: read: connection refused",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.007s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x3db417569280>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get "https://acme-staging-v02.api.letsencrypt.org/directory": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452->172.30.0.10:53: read: connection refused
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:54452->172.30.0.10:53: read: connection refused",
    }
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
21:30:12 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T22:59:02Z
  Ended:       2026-10-14T23:18:03Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j7558016-svc", component="backend", name="actualhostedcluster", region="westus3", severity="warning"
  Description: Backend controller workqueue actualhostedcluster has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 13m 26s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 50b7ccf · merge 6548cfe
 centralus e2e failed #7527 (merged) 1
e2e (1)
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift
20:56:04 UTC · failed at e2e · test Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster
Unexpected error:
    <*fmt.wrapError | 0x3d5895186200>: 
    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster="np-autoscale-cluster" in resourcegroup="np-autoscaling-knlckq8m42ph" to finish creating: GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068",
      "name": "b31da953-f506-41e7-9fc1-8b942d4e8068",
      "status": "Failed",
      "startTime": "2026-10-14T21:44:40.287298623Z",
      "endTime": "2026-10-14T22:13:42.52978598Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, control-plane-pki-operator, openshift-controller-manager, dns-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, openshift-apiserver, hosted-cluster-config-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068
            correlation ID: 4b021430-3a3e-4288-80c7-fd56abc38462
            client request ID: bf735dc6-c8ef-42d7-978e-a9ed4b21e87f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster
Unexpected error:
    <*fmt.wrapError | 0x3d5895186200>: 
    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster="np-autoscale-cluster" in resourcegroup="np-autoscaling-knlckq8m42ph" to finish creating: GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068",
      "name": "b31da953-f506-41e7-9fc1-8b942d4e8068",
      "status": "Failed",
      "startTime": "2026-10-14T21:44:40.287298623Z",
      "endTime": "2026-10-14T22:13:42.52978598Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-network-operator, catalog-operator, packageserver, openshift-route-controller-manager, cluster-policy-controller, cluster-storage-operator, control-plane-pki-operator, openshift-controller-manager, dns-operator, kube-controller-manager, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, csi-snapshot-controller-operator, machine-approver, azure-cloud-controller-manager, cluster-autoscaler, openshift-apiserver, hosted-cluster-config-operator, redhat-marketplace-catalog, community-operators-catalog, cluster-version-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7016064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b31da953-f506-41e7-9fc1-8b942d4e8068
            correlation ID: 4b021430-3a3e-4288-80c7-fd56abc38462
            client request ID: bf735dc6-c8ef-42d7-978e-a9ed4b21e87f
occurred
2h 17m 5s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head f1d1b25 · merge f1d1b25
 centralus alert, e2e failed #7538 (merged) 2
e2e (1)
failed to create nodepool <nodepool>: failed waiting for nodepool="<nodepool>" for cluster "<cluster>" in resourcegroup="<resource-group>" to finish creating: GET <url>
20:54:04 UTC · failed at e2e · test Customer should be able to delete a nodepool · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x3ad81c033ca0>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-gsxwfjg2wp8q" to finish creating: GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47",
      "name": "b803943b-9e39-4dba-aaa8-6afe2e02cb47",
      "status": "Failed",
      "startTime": "2026-10-14T22:01:54.508607559Z",
      "endTime": "2026-10-14T22:30:54.403314647Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47
            correlation ID: 988bdfac-593d-495e-9f75-b7aaa11925ed
            client request ID: 251c1e65-e123-4756-a69e-5f6923040f9c
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x3ad81c033ca0>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-gsxwfjg2wp8q" to finish creating: GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47",
      "name": "b803943b-9e39-4dba-aaa8-6afe2e02cb47",
      "status": "Failed",
      "startTime": "2026-10-14T22:01:54.508607559Z",
      "endTime": "2026-10-14T22:30:54.403314647Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gsxwfjg2wp8q--managed/providers/Microsoft.Compute/virtualMachines/u0w4f2e8s8w0a8w-np-delete-main-tbj7t-gshqg to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6056960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b803943b-9e39-4dba-aaa8-6afe2e02cb47
            correlation ID: 988bdfac-593d-495e-9f75-b7aaa11925ed
            client request ID: 251c1e65-e123-4756-a69e-5f6923040f9c
alert (1)
alert [svc] LeaderElectionLeaseStale fired
20:54:04 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T22:51:50Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j6056960-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.176:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6056960/providers/microsoft.monitor/accounts/services-j6056960", namespace="klusterlet-2teqmhjn22ojhoa0ihg0gcmdrkaa544t", pod="arohcp-monitor-kube-state-metrics-7988dc648c-lrm27", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2teqmhjn22ojhoa0ihg0gcmdrkaa544t on cluster ci01-j6056960-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
2h 19m 40s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head d3c31a8
 centralus alert, e2e failed #6866 (open) 2
e2e (1)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift
20:12:13 UTC · failed at e2e · test Customer should be able to create node pools with ARM64-based VMs · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool "arm64-vm-np-1" with VM size "Standard_D2plds_v6"
Unexpected error:
    <*fmt.wrapError | 0x5bcdb5a5440>: 
    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool="arm64-vm-np-1" for cluster "arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-5gxqdd8p2qbn" to finish creating: GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e",
      "name": "9da300d2-61ca-49b6-8cec-c2ecaffe701e",
      "status": "Failed",
      "startTime": "2026-10-14T21:26:38.348499296Z",
      "endTime": "2026-10-14T21:55:39.3330542Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e
            correlation ID: 7d23b8d5-7dba-484d-af3a-9becf223a3c1
            client request ID: 66e7e251-c1da-4383-aef0-9b31e92742c7
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool "arm64-vm-np-1" with VM size "Standard_D2plds_v6"
Unexpected error:
    <*fmt.wrapError | 0x5bcdb5a5440>: 
    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool="arm64-vm-np-1" for cluster "arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-5gxqdd8p2qbn" to finish creating: GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e",
      "name": "9da300d2-61ca-49b6-8cec-c2ecaffe701e",
      "status": "Failed",
      "startTime": "2026-10-14T21:26:38.348499296Z",
      "endTime": "2026-10-14T21:55:39.3330542Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3055232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9da300d2-61ca-49b6-8cec-c2ecaffe701e
            correlation ID: 7d23b8d5-7dba-484d-af3a-9becf223a3c1
            client request ID: 66e7e251-c1da-4383-aef0-9b31e92742c7
occurred
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
20:12:13 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T21:26:14Z
  Ended:       2026-10-14T21:46:14Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j3055232-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-14T21:31:13Z
  Ended:       2026-10-14T21:46:14Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j3055232-svc", component="backend", name="csstatedump", region="centralus", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 17m 36s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head a3fcc0c · merge 4e38817
 centralus alert, e2e failed ★#7502 (merged) 2
e2e (1)
| take 101, Post "<url>": context deadline exceeded
19:41:51 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x15bb4b38ec0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:28:59.1453968Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:28:59.1453968Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x15bb4b38e80>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:28:59.1453968Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x15bb47fe640>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x15bb4b38e60>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:28:59.1453968Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x15bb6818240>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x15bb4b38ec0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:28:59.1453968Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:28:59.1453968Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x15bb4b38e80>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:28:59.1453968Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x15bb47fe640>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x15bb4b38e60>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-b7t69sxhzhzq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:28:59.1453968Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x15bb6818240>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
19:41:51 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T21:02:26Z
  Ended:       2026-10-14T21:08:28Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j8112128-svc", component="backend", name="actualhostedcluster", region="centralus", severity="warning"
  Description: Backend controller workqueue actualhostedcluster has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 22m 45s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 814133c
 centralus e2e failed ★#7498 (open) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
19:27:06 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x21b5046e260>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:21:27.9443881Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:27.9443881Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x21b5046e220>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:27.9443881Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x21b4fae81e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x21b5046e1e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:27.9443881Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x21b5141a0c0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x21b5046e260>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:21:27.9443881Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:27.9443881Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x21b5046e220>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:27.9443881Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x21b4fae81e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x21b5046e1e0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vwplkrs88dgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:27.9443881Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x21b5141a0c0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 33m 41s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 868faef
 westus3 alert, e2e failed ★#7497 (open) 3
e2e (1)
| take 101, Post "<url>": context deadline exceeded
19:26:58 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1b7133f5ef60>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:36:39.12753Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:36:39.12753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1b7133f5ef40>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:36:39.12753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1b7133bfcb40>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1b7133f5ef00>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:36:39.12753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1b7135c91110>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1b7133f5ef60>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:36:39.12753Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:36:39.12753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1b7133f5ef40>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:36:39.12753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1b7133bfcb40>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1b7133f5ef00>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ncn9lck5zdr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:36:39.12753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1b7135c91110>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
alert (2)
alert [svc] FrontendPathLatency fired
19:26:58 UTC · failed at alert · test [aro-hcp-observability] [svc] alert FrontendPathLatency does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T20:45:07Z
  Ended:       2026-10-14T20:56:08Z
  Severity:    Sev3
  Labels:      alertname="FrontendPathLatency", cluster="ci01-j2147200-svc", component="frontend", method="put", region="westus3", route="/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}", severity="warning"
  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2147200-svc).
Firing 2:
  State:       Resolved
  Started:     2026-10-14T21:06:15Z
  Ended:       2026-10-14T21:18:04Z
  Severity:    Sev3
  Labels:      alertname="FrontendPathLatency", cluster="ci01-j2147200-svc", component="frontend", method="put", region="westus3", route="/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}", severity="warning"
  Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2147200-svc).
alert [svc] LeaderElectionLeaseStale fired
19:26:58 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T21:31:37Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j2147200-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.205:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2147200/providers/microsoft.monitor/accounts/services-j2147200", namespace="klusterlet-2tepk3mcjhubmd4d9nfbho173hml93jh", pod="arohcp-monitor-kube-state-metrics-7988dc648c-jk4qm", prometheus="prometheus/prometheus", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tepk3mcjhubmd4d9nfbho173hml93jh on cluster ci01-j2147200-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
2h 19m 52s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 19b136f
 westus3 e2e failed ★#7500 (open) 2
e2e (2)
| take 101, Post "<url>": context deadline exceeded
19:26:49 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3e937aeae7a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:21:35.1284753Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:35.1284753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3e937aeae780>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:35.1284753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3e937a6e6640>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3e937aeae700>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:35.1284753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3e937c0f9860>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3e937aeae7a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T20:21:35.1284753Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:35.1284753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3e937aeae780>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:35.1284753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3e937a6e6640>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3e937aeae700>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fzv524xhzt56/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T20:21:35.1284753Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3e937c0f9860>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
cluster <resource-group>/<cluster> entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
19:26:49 UTC · failed at e2e · test ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-wz659986bvj7/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-wz659986bvj7/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
2h 18m 19s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head f026aec · merge d13cfae
 centralus e2e failed ⚠#6746 (merged) 1
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in <url> context deadline exceeded
19:26:43 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x32bc856472e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x32bc85647260>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x32bc856472e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2t8nlzw5hxlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x32bc85647260>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
2h 12m 19s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head c171e40 · merge c171e40
 centralus e2e failed ★#7519 (merged) 1
e2e (1)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist; provider Microsoft.RedHatOpenShift
19:26:43 UTC · failed at e2e · test Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x190aa1492400>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-qbnc288qgj6j" to finish creating: GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7",
      "name": "199b8626-fe4e-4b24-9607-bad9bdfbcbe7",
      "status": "Failed",
      "startTime": "2026-10-14T20:29:55.420741799Z",
      "endTime": "2026-10-14T20:59:01.637394871Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-6rhdr-cwgtg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-6rhdr-cwgtg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7
            correlation ID: 1367eb49-5332-4cf5-9a4a-4b806be9f934
            client request ID: 9a4866d1-15fd-4118-b349-e831018d7dac
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x190aa1492400>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-qbnc288qgj6j" to finish creating: GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7",
      "name": "199b8626-fe4e-4b24-9607-bad9bdfbcbe7",
      "status": "Failed",
      "startTime": "2026-10-14T20:29:55.420741799Z",
      "endTime": "2026-10-14T20:59:01.637394871Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-6rhdr-cwgtg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-6rhdr-cwgtg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-qbnc288qgj6j--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-6rhdr-cwgtg to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3865088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199b8626-fe4e-4b24-9607-bad9bdfbcbe7
            correlation ID: 1367eb49-5332-4cf5-9a4a-4b806be9f934
            client request ID: 9a4866d1-15fd-4118-b349-e831018d7dac
occurred
2h 21m 1s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head bb4f900 · merge bb4f900
 westus3 e2e failed ★#7509 (merged) 1
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in <url> context deadline exceeded
19:26:40 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2ba1a496c2e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x2ba1a496c2c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2ba1a496c2e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p5dbqs5dbw4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x2ba1a496c2c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
42m 10s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 5bf7c6e · merge 5bf7c6e
 centralus provision failed #7492 (merged) 1
provision (1)
ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '<vault-name>' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link <url>
19:26:29 UTC · failed at provision · test Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2) · suite templatize-pipeline
Show raw failure
time=2026-10-14T20:02:04.703Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description="Step infra\n  Kind: ARM\n  Template: templates/mgmt-infra.bicep\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam"
time=2026-10-14T20:02:05.214Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2
time=2026-10-14T20:02:05.894Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0582272-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79
time=2026-10-14T20:03:06.110Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err="stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/c992d66eb9a38cb8bacf66c4633734326584f1d12bbb6c98da67de8819d3eb79/operationStatuses/08584095987599895937\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/mgmt-kv-2awbxe73l63vi\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n            \\\"message\\\": \\\"The vault name 'ah-ci01-mg-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/msi-kv-g2p3wnorghzau\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n            \\\"message\\\": \\\"The vault name 'ah-ci01-mi-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0582272-mgmt-2/providers/Microsoft.Resources/deployments/ah-ci01-me-j0582272-2\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n            \\\"message\\\": \\\"The vault name 'ah-ci01-me-j0582272-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"
2h 15m 5s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 664258c
 centralus e2e failed ⚠#7499 (open) 1
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in <url> context deadline exceeded
19:26:26 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x298032100120>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x298032100100>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x298032100120>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mpzr4p97ncr4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x298032100100>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
2h 32m 16s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head cf7a1e9 · merge cf7a1e9
 westus3 alert, e2e failed #7529 (merged) 2
e2e (1)
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version <version> is Partial (want Completed), started <duration> ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift
19:09:11 UTC · failed at e2e · test Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0x15cfa7358820>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-kkjsbcrkrfqs" to finish creating: GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d",
      "name": "3d48f5df-4419-42d0-908d-f55c9e58b43d",
      "status": "Failed",
      "startTime": "2026-10-14T20:00:20.459546459Z",
      "endTime": "2026-10-14T20:29:21.27372498Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 24m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d
            correlation ID: 7a9ff375-e54e-4cef-a85f-da37d27bb541
            client request ID: 20f4c917-baec-45d6-a1e1-7f0ef2326c1c
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster "private-kv-cluster" with private keyvault
Unexpected error:
    <*fmt.wrapError | 0x15cfa7358820>: 
    failed waiting for cluster="private-kv-cluster" in resourcegroup="private-keyvault-kkjsbcrkrfqs" to finish creating: GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d",
      "name": "3d48f5df-4419-42d0-908d-f55c9e58b43d",
      "status": "Failed",
      "startTime": "2026-10-14T20:00:20.459546459Z",
      "endTime": "2026-10-14T20:29:21.27372498Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 24m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3670144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d48f5df-4419-42d0-908d-f55c9e58b43d
            correlation ID: 7a9ff375-e54e-4cef-a85f-da37d27bb541
            client request ID: 20f4c917-baec-45d6-a1e1-7f0ef2326c1c
occurred
alert (1)
alert [hcp] KubePodNotReady fired
19:09:11 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubePodNotReady does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T20:10:46Z
  Ended:       2026-10-14T20:41:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3670144-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tep2lb9i2eso8etu810kd5rk9tkuprr-a1f8l5v2m8b4i0a", pod="router-84bfd8949d-rj9jm", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2tep2lb9i2eso8etu810kd5rk9tkuprr-a1f8l5v2m8b4i0a/router-84bfd8949d-rj9jm has been in a non-ready state for longer than 5 minutes.
2h 15m 6s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 5548820
 centralus alert, e2e failed #6635 (open) 2
e2e (1)
ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift
19:01:26 UTC · failed at e2e · test ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:320]: Unexpected error:
    <*errors.joinError | 0x1e1ed182150>: 
    resource group "rg-candidate-4-21-xnx2svpmndkd": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-21-bqkz5m" in resourcegroup="rg-candidate-4-21-xnx2svpmndkd" to finish deleting: GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7",
      "name": "aefc9a01-d084-42f4-9d9d-673831774ba7",
      "status": "Failed",
      "startTime": "2026-10-14T20:07:21.117969363Z",
      "endTime": "2026-10-14T20:31:23.904548294Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7
            correlation ID: e7f218c6-4d7a-4ce1-8044-c7ec05ec7356
            client request ID: e554513d-44da-4c58-8dae-c72863b0374d
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:320]: Unexpected error:
    <*errors.joinError | 0x1e1ed182150>: 
    resource group "rg-candidate-4-21-xnx2svpmndkd": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-21-bqkz5m" in resourcegroup="rg-candidate-4-21-xnx2svpmndkd" to finish deleting: GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7",
      "name": "aefc9a01-d084-42f4-9d9d-673831774ba7",
      "status": "Failed",
      "startTime": "2026-10-14T20:07:21.117969363Z",
      "endTime": "2026-10-14T20:31:23.904548294Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6397952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aefc9a01-d084-42f4-9d9d-673831774ba7
            correlation ID: e7f218c6-4d7a-4ce1-8044-c7ec05ec7356
            client request ID: e554513d-44da-4c58-8dae-c72863b0374d
occurred
alert (1)
alert [hcp] KubeDeploymentReplicasMismatch fired
19:01:26 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T20:44:23Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6397952-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.193:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6397952/providers/microsoft.monitor/accounts/hcps-j6397952", namespace="ocm-arohcpci01-2teosscr06nia6i1ff9khrnv5kp7a288-t4e9t7g2o3k6j4x", pod="arohcp-monitor-kube-state-metrics-7988dc648c-cl786", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teosscr06nia6i1ff9khrnv5kp7a288-t4e9t7g2o3k6j4x/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T20:44:23Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6397952-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.193:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6397952/providers/microsoft.monitor/accounts/hcps-j6397952", namespace="ocm-arohcpci01-2teosscr06nia6i1ff9khrnv5kp7a288-t4e9t7g2o3k6j4x", pod="arohcp-monitor-kube-state-metrics-7988dc648c-cl786", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teosscr06nia6i1ff9khrnv5kp7a288-t4e9t7g2o3k6j4x/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
2h 12m 32s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 413c69d · merge 09587f6
 westus3 e2e failed ★#7440 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
18:54:11 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3efda7f42600>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T19:38:56.9987101Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T19:38:56.9987101Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3efda7f425e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T19:38:56.9987101Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3efda78045a0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3efda7f425c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T19:38:56.9987101Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3efda91811a0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x3efda7f42600>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T19:38:56.9987101Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T19:38:56.9987101Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x3efda7f425e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T19:38:56.9987101Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x3efda78045a0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x3efda7f425c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tpsrv9dgf7c8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T19:38:56.9987101Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x3efda91811a0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
1h 25m 15s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head effc7fa
 westus3 alert, provision failed ⚠#6946 (open) 2
provision (1)
error running Shell Step, failed to execute shell command: {"level":"INFO","msg":"starting cluster reconciliation","component":"aks-cluster-create","subscriptionID":"<redacted-id>","resourceGroup":"hcp-underlay-ci01-j2981888-mgmt-1","clusterName":"ci01-j298188
18:45:15 UTC · failed at provision · test Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1) · suite templatize-pipeline
Show raw failure
time=2026-10-14T19:33:50.044Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description="Step cluster-create\n  Kind: Shell\n  Command: ./aks-cluster-create\n"
time=2026-10-14T20:03:49.640Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err="stamp 1: error running Shell Step, failed to execute shell command: {\"time\":\"2026-10-14T19:33:50.055321564Z\",\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j2981888-mgmt-1\",\"clusterName\":\"ci01-j2981888-mgmt-1\",\"region\":\"westus3\"}\n{\"time\":\"2026-10-14T19:33:52.298968498Z\",\"level\":\"INFO\",\"msg\":\"configured pools\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j2981888-mgmt-1\",\"clusterName\":\"ci01-j2981888-mgmt-1\",\"region\":\"westus3\",\"system\":1,\"worker\":3,\"infra\":2}\n{\"time\":\"2026-10-14T19:33:52.677544841Z\",\"level\":\"INFO\",\"msg\":\"deploying configured resources\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j2981888-mgmt-1\",\"clusterName\":\"ci01-j2981888-mgmt-1\",\"region\":\"westus3\",\"deployment\":\"aks-cluster-create\",\"poolCount\":6,\"clusterFields\":null}\n signal: killed"
alert (1)
alert [svc] MaestroAgentMetricsProxyDown fired
18:45:15 UTC · failed at alert · test [aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T19:52:31Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j2981888-mgmt-2", component="monitoring-infrastructure", region="westus3", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.
45m 41s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head f026aec · merge d13cfae
 centralus provision failed #6746 (merged) 1
provision (1)
ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network
18:34:47 UTC · failed at provision · test Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2) · suite templatize-pipeline
Show raw failure
time=2026-10-14T19:14:18.297Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description="Step nsp\n  Kind: ARM\n  Template: templates/mgmt-nsp.bicep\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam"
time=2026-10-14T19:14:18.788Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2
time=2026-10-14T19:14:19.241Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=cf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9264896-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31
time=2026-10-14T19:14:49.370Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err="stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Resources/deployments/cf8c9a34812ddee83e87f5e1fcb73992e7f0046ac9c8abee6011c27a19ac7c31/operationStatuses/08584096016264886235\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Resources/deployments/profile-xpjnxfpmcvora\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9264896-mgmt-2/resourceAssociations/nsp-j9264896-mgmt-2-ra-a9c395a5-2fe0-5927-97a4-6856e96269ae\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"HttpRequestException\\\",\\r\\n                \\\"message\\\": \\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9264896-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9264896-mgmt-2/resourceAssociations/nsp-j9264896-mgmt-2-ra-a9c395a5-2fe0-5927-97a4-6856e96269ae, Error message: An error occurred while sending the request.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"
2h 31m 55s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 8af0205 · merge 8af0205
 centralus e2e failed #7528 (merged) 1
e2e (1)
VerifySimpleWebApp failed: DNS for route host <route-host> did not resolve: DNS for <route-host> did not resolve within <duration> (last error: lookup <host> on <dns-server>: dial udp <ip>: i/o timeout): context deadline exceeded
18:32:20 UTC · failed at e2e · test Customer should be able to create a HCP cluster and use cilium CNI plugin · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI
Unexpected error:
    <*errors.joinError | 0x8382000a060>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x8381fe06080>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x8381fe064a0>{
                    msg: "DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x8381fe06480>{
                        msg: "DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI
Unexpected error:
    <*errors.joinError | 0x8382000a060>: 
    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x8381fe06080>{
                msg: "VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                err: <*fmt.wrapError | 0x8381fe064a0>{
                    msg: "DNS for route host agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                    err: <*fmt.wrapError | 0x8381fe06480>{
                        msg: "DNS for agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67tj5.apps.aro.cilium-cl.sngp.j3256704.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
                        err: <context.deadlineExceededError>{},
                    },
                },
            },
        ],
    }
occurred
2h 12m 45s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head b6c0794 · merge b6c0794
 centralus alert, e2e failed #7524 (merged) 3
e2e (1)
ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/<uuid>' under resource group '' was not found.; provider Microsoft.RedHatOpenShift
18:26:44 UTC · failed at e2e · test Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk
Unexpected error:
    <*fmt.wrapError | 0x12acc23ba300>: 
    failed to create NodePool ephemeral-np: failed waiting for nodepool="ephemeral-np" for cluster "ephemeral-disk" in resourcegroup="ephemeral-osdisk-kwgn8mlj4cqw" to finish creating: GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: ResourceNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "ResourceNotFound",
        "message": "The resource 'locations/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64' under resource group '' was not found.",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64
            correlation ID: 6cf5166e-d7f7-4d42-8f52-96bfabf5f7df
            client request ID: d4b2a1df-961f-48a7-9e13-886e82b9c210
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk
Unexpected error:
    <*fmt.wrapError | 0x12acc23ba300>: 
    failed to create NodePool ephemeral-np: failed waiting for nodepool="ephemeral-np" for cluster "ephemeral-disk" in resourcegroup="ephemeral-osdisk-kwgn8mlj4cqw" to finish creating: GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: ResourceNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "ResourceNotFound",
        "message": "The resource 'locations/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64' under resource group '' was not found.",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2922112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/431cc2e5-1210-4f39-b481-aac97e02bf64
            correlation ID: 6cf5166e-d7f7-4d42-8f52-96bfabf5f7df
            client request ID: d4b2a1df-961f-48a7-9e13-886e82b9c210
occurred
alert (2)
alert [svc] LeaderElectionLeaseStale fired
18:26:44 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T20:10:46Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j2922112-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.238:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2922112/providers/microsoft.monitor/accounts/services-j2922112", namespace="klusterlet-2teodcofsg9i8mg4vfv72dcg8ci8ollu", pod="arohcp-monitor-kube-state-metrics-7988dc648c-2wm6l", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2teodcofsg9i8mg4vfv72dcg8ci8ollu on cluster ci01-j2922112-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
alert [hcp] KubeDeploymentReplicasMismatch fired
18:26:44 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T20:01:44Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2922112-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.238:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2922112/providers/microsoft.monitor/accounts/hcps-j2922112", namespace="ocm-arohcpci01-2teodb2c2csmhm2lduce3cgjjgh1cnh8-n9y2x5u9j4c0t2r", pod="arohcp-monitor-kube-state-metrics-7988dc648c-2wm6l", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teodb2c2csmhm2lduce3cgjjgh1cnh8-n9y2x5u9j4c0t2r/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T20:01:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2922112-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.238:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2922112/providers/microsoft.monitor/accounts/hcps-j2922112", namespace="ocm-arohcpci01-2teodb2c2csmhm2lduce3cgjjgh1cnh8-n9y2x5u9j4c0t2r", pod="arohcp-monitor-kube-state-metrics-7988dc648c-2wm6l", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teodb2c2csmhm2lduce3cgjjgh1cnh8-n9y2x5u9j4c0t2r/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
2h 3m 40s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 952fb40 · merge 8d693ac
 centralus n/a passed #6968 (merged) 0 n/a
2h 7m 33s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 5946afb
 centralus e2e failed ⚠#7477 (closed) 8
e2e (8)
failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T19:48:15
18:22:06 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.007s.
Expected success, but got an error:
    <*fmt.wrapError | 0x356d91091ca0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);
    let _endTime = datetime(2026-10-14T19:48:15.7310678Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-vcw9kpw6mcqc'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T19:48:28.4804395Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e", ActivityId:"9181dd6d-5c5a-470f-8a62-6f1f2abde29b", SubActivityId:"9181dd6d-5c5a-470f-8a62-6f1f2abde29b", ActivityType:"GW.Http.CallContext", ParentActivityId:"9181dd6d-5c5a-470f-8a62-6f1f2abde29b", ActivityStack:"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x356d921ef170>{
            errs: [
                <*fmt.wrapError | 0x356d91091c80>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x356d91091c60>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x356d91091b60>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x356d91091b20>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x356d9135a600>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.007s.
Expected success, but got an error:
    <*fmt.wrapError | 0x356d91091ca0>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);
    let _endTime = datetime(2026-10-14T19:48:15.7310678Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-vcw9kpw6mcqc'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T19:48:28.4804395Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e", ActivityId:"9181dd6d-5c5a-470f-8a62-6f1f2abde29b", SubActivityId:"9181dd6d-5c5a-470f-8a62-6f1f2abde29b", ActivityType:"GW.Http.CallContext", ParentActivityId:"9181dd6d-5c5a-470f-8a62-6f1f2abde29b", ActivityStack:"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x356d921ef170>{
            errs: [
                <*fmt.wrapError | 0x356d91091c80>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x356d91091c60>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x356d91091b60>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x356d91091b20>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T19:48:15.7310676Z);\nlet _endTime = datetime(2026-10-14T19:48:15.7310678Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-vcw9kpw6mcqc'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T19:48:28.4804395Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e\", ActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", SubActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9181dd6d-5c5a-470f-8a62-6f1f2abde29b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;7eea97f8-e218-4155-b4d6-a3666cff474e ARID=9181dd6d-5c5a-470f-8a62-6f1f2abde29b > GW.Http.CallContext/9181dd6d-5c5a-470f-8a62-6f1f2abde29b)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x356d9135a600>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
failed to create nodepool <nodepool>: failed waiting for nodepool="<nodepool>" for cluster "<cluster>" in resourcegroup="<resource-group>" to finish creating: GET <url>
18:22:06 UTC · failed at e2e · test Customer should be able to delete a nodepool · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x235e937976c0>: 
    failed to create NodePool np-deletable: failed waiting for nodepool="np-deletable" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-68hx6zjhrzrm" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14b98349-fe00-4d10-b2ba-bdfa836e3ef6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14b98349-fe00-4d10-b2ba-bdfa836e3ef6",
      "name": "14b98349-fe00-4d10-b2ba-bdfa836e3ef6",
      "status": "Failed",
      "startTime": "2026-10-14T19:36:58.429408765Z",
      "endTime": "2026-10-14T20:05:59.913923951Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-68hx6zjhrzrm--managed/providers/Microsoft.Compute/virtualMachines/n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-68hx6zjhrzrm--managed/providers/Microsoft.Compute/virtualMachines/n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14b98349-fe00-4d10-b2ba-bdfa836e3ef6
            correlation ID: b0044d15-3081-45ef-b76a-eef2cf552d58
            client request ID: 9699734b-7b5f-4b49-8b9d-898444b64f0d
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x235e937976c0>: 
    failed to create NodePool np-deletable: failed waiting for nodepool="np-deletable" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-68hx6zjhrzrm" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14b98349-fe00-4d10-b2ba-bdfa836e3ef6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14b98349-fe00-4d10-b2ba-bdfa836e3ef6",
      "name": "14b98349-fe00-4d10-b2ba-bdfa836e3ef6",
      "status": "Failed",
      "startTime": "2026-10-14T19:36:58.429408765Z",
      "endTime": "2026-10-14T20:05:59.913923951Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-68hx6zjhrzrm--managed/providers/Microsoft.Compute/virtualMachines/n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-68hx6zjhrzrm--managed/providers/Microsoft.Compute/virtualMachines/n6f3b4y4h9b0a8m-np-deletable-v7kcg-vdl7x to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14b98349-fe00-4d10-b2ba-bdfa836e3ef6
            correlation ID: b0044d15-3081-45ef-b76a-eef2cf552d58
            client request ID: 9699734b-7b5f-4b49-8b9d-898444b64f0d
timeout during CreateHCPClusterAndWait; context deadline exceeded
18:22:06 UTC · failed at e2e · test Customer should be able to create an HCP cluster with back-level version 4.20 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x1399efb46de0>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-2n7cffz9j9vb" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x1399ef809d30>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x1399efb46de0>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-2n7cffz9j9vb" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-2n7cffz9j9vb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x1399ef809d30>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-2n7cffz9j9vb",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
18:22:06 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x2e1d6333c380>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-9dqj7t6blpw4" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2",
      "name": "4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:43.252392351Z",
      "endTime": "2026-10-14T19:38:50.14396842Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2
            correlation ID: c406a5fd-e90a-4a6c-93f6-9a6fc3c3e5f6
            client request ID: 746327fb-5483-438f-979e-0881d72f91fd
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x2e1d6333c380>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-9dqj7t6blpw4" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2",
      "name": "4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:43.252392351Z",
      "endTime": "2026-10-14T19:38:50.14396842Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4f4df5a4-fbf5-46b0-8b08-f2e3d3bd5ec2
            correlation ID: c406a5fd-e90a-4a6c-93f6-9a6fc3c3e5f6
            client request ID: 746327fb-5483-438f-979e-0881d72f91fd
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
18:22:06 UTC · failed at e2e · test Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-mxm2nz" with version 4.21.36 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x2e81b57e6100>: 
    failed to create HCP cluster cluster-zstream-4-21-mxm2nz: failed waiting for cluster="cluster-zstream-4-21-mxm2nz" in resourcegroup="rg-zstream-upgrade-4-21-9f22s7ghtq4x" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5",
      "name": "e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:23.643995805Z",
      "endTime": "2026-10-14T19:38:30.41816485Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5
            correlation ID: 40e10400-2a72-4df0-a76e-c12b12d5889c
            client request ID: 5416d7ee-a6cc-4125-9004-a1fe5836d05b
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-mxm2nz" with version 4.21.36 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x2e81b57e6100>: 
    failed to create HCP cluster cluster-zstream-4-21-mxm2nz: failed waiting for cluster="cluster-zstream-4-21-mxm2nz" in resourcegroup="rg-zstream-upgrade-4-21-9f22s7ghtq4x" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5",
      "name": "e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:23.643995805Z",
      "endTime": "2026-10-14T19:38:30.41816485Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e22f032f-fd1f-4f7a-a0f0-79ca0d681ff5
            correlation ID: 40e10400-2a72-4df0-a76e-c12b12d5889c
            client request ID: 5416d7ee-a6cc-4125-9004-a1fe5836d05b
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
18:22:06 UTC · failed at e2e · test SRE starts a control plane z-stream upgrade with an SRE pin · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster "sre-version-pin-vc9jq7" at 4.21.36
Unexpected error:
    <*fmt.wrapError | 0x2da62fe74520>: 
    failed to create HCP cluster sre-version-pin-vc9jq7: failed waiting for cluster="sre-version-pin-vc9jq7" in resourcegroup="admin-api-version-pin-ncqjrnsn47gn" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac",
      "name": "3a2cf661-5481-4ce5-9c2c-34f8ee3716ac",
      "status": "Failed",
      "startTime": "2026-10-14T19:11:51.389728795Z",
      "endTime": "2026-10-14T19:41:00.041259206Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac
            correlation ID: 3bd031da-51c1-4750-a9ad-a1b50d360688
            client request ID: c18e0ef2-a2d7-4b12-a856-0335c427f46a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster "sre-version-pin-vc9jq7" at 4.21.36
Unexpected error:
    <*fmt.wrapError | 0x2da62fe74520>: 
    failed to create HCP cluster sre-version-pin-vc9jq7: failed waiting for cluster="sre-version-pin-vc9jq7" in resourcegroup="admin-api-version-pin-ncqjrnsn47gn" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac",
      "name": "3a2cf661-5481-4ce5-9c2c-34f8ee3716ac",
      "status": "Failed",
      "startTime": "2026-10-14T19:11:51.389728795Z",
      "endTime": "2026-10-14T19:41:00.041259206Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3a2cf661-5481-4ce5-9c2c-34f8ee3716ac
            correlation ID: 3bd031da-51c1-4750-a9ad-a1b50d360688
            client request ID: c18e0ef2-a2d7-4b12-a856-0335c427f46a
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
18:22:06 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x3dad90af33a0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-sbj86zcf4vq4" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d",
      "name": "18cf34f6-b63a-4010-8930-865b2da0c38d",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:44.204708518Z",
      "endTime": "2026-10-14T19:38:50.032848393Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d
            correlation ID: 8581e472-de79-4013-9c83-cf00277e2806
            client request ID: 3009aa8f-ca8e-40df-af26-d6a54c07c012
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x3dad90af33a0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-sbj86zcf4vq4" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d",
      "name": "18cf34f6-b63a-4010-8930-865b2da0c38d",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:44.204708518Z",
      "endTime": "2026-10-14T19:38:50.032848393Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/18cf34f6-b63a-4010-8930-865b2da0c38d
            correlation ID: 8581e472-de79-4013-9c83-cf00277e2806
            client request ID: 3009aa8f-ca8e-40df-af26-d6a54c07c012
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
18:22:06 UTC · failed at e2e · test Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-r82p8j" with version 4.20.41 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x29995a6feda0>: 
    failed to create HCP cluster cluster-zstream-4-20-r82p8j: failed waiting for cluster="cluster-zstream-4-20-r82p8j" in resourcegroup="rg-zstream-upgrade-4-20-dth4ztp5pfvf" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/adc9f47d-7daf-43fd-802a-0b56f4b2fe20
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/adc9f47d-7daf-43fd-802a-0b56f4b2fe20",
      "name": "adc9f47d-7daf-43fd-802a-0b56f4b2fe20",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:44.104891528Z",
      "endTime": "2026-10-14T19:38:50.195897659Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/adc9f47d-7daf-43fd-802a-0b56f4b2fe20
            correlation ID: c3ecc718-6d60-4363-8871-f88a84d57267
            client request ID: acba8711-1e99-4e32-9874-7b4219569054
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-r82p8j" with version 4.20.41 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x29995a6feda0>: 
    failed to create HCP cluster cluster-zstream-4-20-r82p8j: failed waiting for cluster="cluster-zstream-4-20-r82p8j" in resourcegroup="rg-zstream-upgrade-4-20-dth4ztp5pfvf" to finish creating: GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/adc9f47d-7daf-43fd-802a-0b56f4b2fe20
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/adc9f47d-7daf-43fd-802a-0b56f4b2fe20",
      "name": "adc9f47d-7daf-43fd-802a-0b56f4b2fe20",
      "status": "Failed",
      "startTime": "2026-10-14T19:09:44.104891528Z",
      "endTime": "2026-10-14T19:38:50.195897659Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6618368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/adc9f47d-7daf-43fd-802a-0b56f4b2fe20
            correlation ID: c3ecc718-6d60-4363-8871-f88a84d57267
            client request ID: acba8711-1e99-4e32-9874-7b4219569054
occurred
58m 1s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 9cb1004
 westus3 provision failed #7133 (open) 1
provision (1)
failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1
18:05:44 UTC · failed at provision · test Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1) · suite templatize-pipeline
Show raw failure
time=2026-10-14T18:50:15.283Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 description="Step mitigations\n Kind: Helm\n"
time=2026-10-14T18:50:15.838Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 values="map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]"
time=2026-10-14T18:50:15.838Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1
time=2026-10-14T18:50:16.026Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 namespace=aro-hcp-mitigations
time=2026-10-14T18:50:16.350Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 dryRun=false
time=2026-10-14T18:50:16.350Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1
time=2026-10-14T18:50:16.477Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1
time=2026-10-14T18:56:25.364Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err="failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"
time=2026-10-14T18:56:25.364Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"
2h 23m 35s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 7325d3a · merge 9f60977
 westus3 n/a passed #7132 (merged) 0 n/a
2h 10m 16s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head c171e40 · merge c171e40
 centralus alert failed #7519 (merged) 1
alert (1)
alert [svc] KubePodNotReady fired
17:09:29 UTC · failed at alert · test [aro-hcp-observability] [svc] alert KubePodNotReady does not fire · suite aro-hcp-tests
Show raw failure
alert fired 70 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-4fnrs", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-4fnrs has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-qd4lc", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-qd4lc has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-q9htv", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-q9htv has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-2v2ll", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-2v2ll has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-p45vd", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-p45vd has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-v2bxl", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-v2bxl has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-rsvx7", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-rsvx7 has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-zw6fj", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-zw6fj has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-r5r7n", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-r5r7n has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-xb9hp", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-xb9hp has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-bvr8s", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-bvr8s has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-vsrr8", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-vsrr8 has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-srv48", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-srv48 has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-q5vds", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-q5vds has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-rjwtc", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-rjwtc has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-nzgpk", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-nzgpk has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-4slr2", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-4slr2 has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-m7ftv", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-m7ftv has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-v7hdm", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-v7hdm has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-86ssh", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-86ssh has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-jp6fj", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-jp6fj has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-xjxzz", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-xjxzz has been in a non-ready state for longer than 5 minutes.
Firing 23:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-b7ph9", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-b7ph9 has been in a non-ready state for longer than 5 minutes.
Firing 24:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-drpfk", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-drpfk has been in a non-ready state for longer than 5 minutes.
Firing 25:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-6q6w9", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-6q6w9 has been in a non-ready state for longer than 5 minutes.
Firing 26:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-77r8t", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-77r8t has been in a non-ready state for longer than 5 minutes.
Firing 27:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-vhdzq", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-vhdzq has been in a non-ready state for longer than 5 minutes.
Firing 28:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-hdcnl", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-hdcnl has been in a non-ready state for longer than 5 minutes.
Firing 29:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-5z589", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-5z589 has been in a non-ready state for longer than 5 minutes.
Firing 30:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-kr492", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-kr492 has been in a non-ready state for longer than 5 minutes.
Firing 31:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-bcl97", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-bcl97 has been in a non-ready state for longer than 5 minutes.
Firing 32:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-s49mm", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-s49mm has been in a non-ready state for longer than 5 minutes.
Firing 33:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-drmdx", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-drmdx has been in a non-ready state for longer than 5 minutes.
Firing 34:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-sm8gg", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-sm8gg has been in a non-ready state for longer than 5 minutes.
Firing 35:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-2xnc2", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-2xnc2 has been in a non-ready state for longer than 5 minutes.
Firing 36:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-7x9gw", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-7x9gw has been in a non-ready state for longer than 5 minutes.
Firing 37:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-bddnr", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-bddnr has been in a non-ready state for longer than 5 minutes.
Firing 38:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-kz42p", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-kz42p has been in a non-ready state for longer than 5 minutes.
Firing 39:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-nz88g", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-nz88g has been in a non-ready state for longer than 5 minutes.
Firing 40:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-4pq7x", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-4pq7x has been in a non-ready state for longer than 5 minutes.
Firing 41:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-flmn9", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-flmn9 has been in a non-ready state for longer than 5 minutes.
Firing 42:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-fwvds", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-fwvds has been in a non-ready state for longer than 5 minutes.
Firing 43:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-mk9q7", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-mk9q7 has been in a non-ready state for longer than 5 minutes.
Firing 44:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-6bqxw", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-6bqxw has been in a non-ready state for longer than 5 minutes.
Firing 45:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-xn8q4", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-xn8q4 has been in a non-ready state for longer than 5 minutes.
Firing 46:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-4s44z", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-4s44z has been in a non-ready state for longer than 5 minutes.
Firing 47:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-fhhv2", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-fhhv2 has been in a non-ready state for longer than 5 minutes.
Firing 48:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-t8v8f", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-t8v8f has been in a non-ready state for longer than 5 minutes.
Firing 49:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-2sg4p", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-2sg4p has been in a non-ready state for longer than 5 minutes.
Firing 50:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-mh74p", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-mh74p has been in a non-ready state for longer than 5 minutes.
Firing 51:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-fgn4b", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-fgn4b has been in a non-ready state for longer than 5 minutes.
Firing 52:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-lq8md", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-lq8md has been in a non-ready state for longer than 5 minutes.
Firing 53:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-wcdz6", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-wcdz6 has been in a non-ready state for longer than 5 minutes.
Firing 54:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-qbfw5", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-qbfw5 has been in a non-ready state for longer than 5 minutes.
Firing 55:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-nshz9", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-nshz9 has been in a non-ready state for longer than 5 minutes.
Firing 56:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-f6nsp", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-f6nsp has been in a non-ready state for longer than 5 minutes.
Firing 57:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-z6t2j", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-z6t2j has been in a non-ready state for longer than 5 minutes.
Firing 58:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-s75xf", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-s75xf has been in a non-ready state for longer than 5 minutes.
Firing 59:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-bmx5q", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-bmx5q has been in a non-ready state for longer than 5 minutes.
Firing 60:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-t57qd", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-t57qd has been in a non-ready state for longer than 5 minutes.
Firing 61:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-4ntcs", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-4ntcs has been in a non-ready state for longer than 5 minutes.
Firing 62:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-cjtgq", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-cjtgq has been in a non-ready state for longer than 5 minutes.
Firing 63:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-6jl7m", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-6jl7m has been in a non-ready state for longer than 5 minutes.
Firing 64:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-79sxk", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-79sxk has been in a non-ready state for longer than 5 minutes.
Firing 65:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-pqnwt", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-pqnwt has been in a non-ready state for longer than 5 minutes.
Firing 66:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-hplkm", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-hplkm has been in a non-ready state for longer than 5 minutes.
Firing 67:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-drfrc", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-drfrc has been in a non-ready state for longer than 5 minutes.
Firing 68:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-9k4dd", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-9k4dd has been in a non-ready state for longer than 5 minutes.
Firing 69:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-4zv7k", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-4zv7k has been in a non-ready state for longer than 5 minutes.
Firing 70:
  State:       Fired (not resolved)
  Started:     2026-10-14T17:56:11Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j6227072-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-5ddf5fbfd4-ql2fg", region="centralus", severity="warning"
  Description: Pod monitoring/kube-events-5ddf5fbfd4-ql2fg has been in a non-ready state for longer than 5 minutes.
2h 29m 17s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head e9f8f97
 westus3 alert, e2e failed #7301 (open) 2
e2e (1)
ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is "uninstalling"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift
16:49:27 UTC · failed at e2e · test Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0xba275880060>: 
    resource group "customer-rg-xns69mfwd2hq": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="basic-hcp-cluster" in resourcegroup="customer-rg-xns69mfwd2hq" to finish deleting: GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387",
      "name": "3176ad81-fee9-4cf2-8688-8a1c24844387",
      "status": "Failed",
      "startTime": "2026-10-14T18:01:05.468018625Z",
      "endTime": "2026-10-14T18:25:10.474406797Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ten52g4l2k3e0ijcet6akeh14ne2nbv still exists (deletion dispatched at 2026-10-14T18:01:06Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387
            correlation ID: fdc30364-8ed7-4835-a286-f7a091278652
            client request ID: a6850f11-48fa-4616-8112-2b5b344c479a
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0xba275880060>: 
    resource group "customer-rg-xns69mfwd2hq": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="basic-hcp-cluster" in resourcegroup="customer-rg-xns69mfwd2hq" to finish deleting: GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387",
      "name": "3176ad81-fee9-4cf2-8688-8a1c24844387",
      "status": "Failed",
      "startTime": "2026-10-14T18:01:05.468018625Z",
      "endTime": "2026-10-14T18:25:10.474406797Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ten52g4l2k3e0ijcet6akeh14ne2nbv still exists (deletion dispatched at 2026-10-14T18:01:06Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6205824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3176ad81-fee9-4cf2-8688-8a1c24844387
            correlation ID: fdc30364-8ed7-4835-a286-f7a091278652
            client request ID: a6850f11-48fa-4616-8112-2b5b344c479a
occurred
alert (1)
alert [hcp] KubeDeploymentReplicasMismatch fired
16:49:27 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T18:43:38Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6205824-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6205824/providers/microsoft.monitor/accounts/hcps-j6205824", namespace="ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6qwtt", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T18:43:38Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6205824-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6205824/providers/microsoft.monitor/accounts/hcps-j6205824", namespace="ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6qwtt", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-14T18:44:39Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6205824-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6205824/providers/microsoft.monitor/accounts/hcps-j6205824", namespace="ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6qwtt", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-14T18:44:39Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6205824-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.196:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6205824/providers/microsoft.monitor/accounts/hcps-j6205824", namespace="ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k", pod="arohcp-monitor-kube-state-metrics-7988dc648c-6qwtt", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2ten52g4l2k3e0ijcet6akeh14ne2nbv-q6i9m8f1j7m6y0k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
2h 15m 37s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head cf7a1e9 · merge cf7a1e9
 westus3 alert, e2e failed ★#7529 (merged) 2
e2e (1)
All ClusterOperators should report Available=True, but these are not available: [console (Available=False)]
16:49:06 UTC · failed at e2e · test Authorized CIDRs Connectivity should allow API access only from authorized VM IP · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 639.547s.
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:
All ClusterOperators should report Available=True, but these are not available: [console (Available=False)]
Expected
    <[]string | len:1, cap:1>: [
        "console (Available=False)",
    ]
to be empty
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 639.547s.
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:
All ClusterOperators should report Available=True, but these are not available: [console (Available=False)]
Expected
    <[]string | len:1, cap:1>: [
        "console (Available=False)",
    ]
to be empty
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
16:49:06 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T18:03:47Z
  Ended:       2026-10-14T18:21:46Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j8253056-svc", component="backend", name="actualhostedcluster", region="westus3", severity="warning"
  Description: Backend controller workqueue actualhostedcluster has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 25m 23s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head ec99735 · merge 6548cfe
 westus3 alert, e2e failed ★#7527 (merged) 2
e2e (1)
| take 101, Post "<url>": context deadline exceeded
16:32:22 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2614e6c7e2e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T17:35:29.8128675Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T17:35:29.8128675Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2614e6c7e2c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T17:35:29.8128675Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2614e65da140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2614e6c7e2a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T17:35:29.8128675Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2614e667e870>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2614e6c7e2e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T17:35:29.8128675Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T17:35:29.8128675Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x2614e6c7e2c0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T17:35:29.8128675Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2614e65da140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x2614e6c7e2a0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jxrgdmqw6vzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T17:35:29.8128675Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x2614e667e870>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
alert (1)
alert [svc] LeaderElectionLeaseStale fired
16:32:22 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T18:44:13Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j8160256-mgmt-2", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.117:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8160256/providers/microsoft.monitor/accounts/services-j8160256", namespace="klusterlet-2temv47eu2tlhk851uh201a18n3tvksi", pod="arohcp-monitor-kube-state-metrics-7988dc648c-jh8gb", prometheus="prometheus/prometheus", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2temv47eu2tlhk851uh201a18n3tvksi on cluster ci01-j8160256-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
1h 55m 14s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head b6c0794 · merge b6c0794
 centralus e2e failed ★#7524 (merged) 2
e2e (2)
cluster <resource-group>/<cluster> entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
16:26:53 UTC · failed at e2e · test ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-v27hgqxjfnl7/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-v27hgqxjfnl7/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
| take 101, Post "<url>": context deadline exceeded
16:26:53 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x23508b9e4c80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T16:58:06.5166082Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:58:06.5166082Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x23508b9e4c20>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:58:06.5166082Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x23508b6cccd0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x23508b9e4c00>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:58:06.5166082Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x23508cfe96e0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x23508b9e4c80>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T16:58:06.5166082Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:58:06.5166082Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x23508b9e4c20>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:58:06.5166082Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x23508b6cccd0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x23508b9e4c00>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gjrlswsvcfc2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:58:06.5166082Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x23508cfe96e0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 15m 53s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 72e7e68
 westus3 n/a passed #7486 (open) 0 n/a
1h 21m 13s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head bb4f900 · merge bb4f900
 centralus alert, provision failed #7509 (merged) 2
provision (1)
failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress context deadline exceeded
15:52:39 UTC · failed at provision · test Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config (stamp=2) · suite templatize-pipeline
Show raw failure
time=2026-10-14T16:56:13.797Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 description="Step deploy-mce-config\n Kind: Helm\n"
time=2026-10-14T16:56:14.254Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 values="map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 global:map[imageOverrides:map[governance_policy_propagator:governance-policy-propagator-acm-216@sha256:deb737b19932d520553c902ee5e0738ebeaad4592e8aa9649c41fec6fc00b62e] registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]"
time=2026-10-14T16:56:14.254Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2
time=2026-10-14T16:56:14.292Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 namespace=multicluster-engine
time=2026-10-14T16:56:14.362Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 dryRun=false
time=2026-10-14T16:56:14.362Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2
time=2026-10-14T16:56:14.432Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2
time=2026-10-14T17:06:42.230Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err="failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\ncontext deadline exceeded"
time=2026-10-14T17:06:42.231Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/multicluster-engine/finalize-mce-config not ready. status: InProgress, message: Job in progress\ncontext deadline exceeded"
alert (1)
alert [svc] MaestroAgentMetricsProxyDown fired
15:52:39 UTC · failed at alert · test [aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T16:54:29Z
  Ended:       2026-10-14T17:00:24Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j8174976-mgmt-1", component="monitoring-infrastructure", region="centralus", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.
2h 24m 34s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head d303552 · merge 8c3c716
 westus3 n/a passed #7383 (merged) 0 n/a
2h 31m 39s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 4bc626a · merge a800194
 westus3 n/a passed #7489 (merged) 0 n/a
43m 0s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 1d23c83
 westus3 provision failed #6364 (open) 1
provision (1)
ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network
15:20:23 UTC · failed at provision · test Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2) · suite templatize-pipeline
Show raw failure
time=2026-10-14T15:57:00.478Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description="Step nsp\n  Kind: ARM\n  Template: templates/mgmt-nsp.bicep\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam"
time=2026-10-14T15:57:01.032Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2
time=2026-10-14T15:57:01.677Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7217792-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8
time=2026-10-14T15:57:32.150Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err="stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Resources/deployments/9e882a024304e709a0349553a748b62ffe4602dc3aa507f98e24a10ee21a2af8/operationStatuses/08584096134641768649\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Resources/deployments/profile-7bczjqgoi25xw\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7217792-mgmt-2/resourceAssociations/nsp-j7217792-mgmt-2-ra-e5d09873-b78f-5e95-bab4-2921347dc438\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"HttpRequestException\\\",\\r\\n                \\\"message\\\": \\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7217792-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7217792-mgmt-2/resourceAssociations/nsp-j7217792-mgmt-2-ra-e5d09873-b78f-5e95-bab4-2921347dc438, Error message: An error occurred while sending the request.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"
2h 16m 7s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 5bf7c6e · merge 5bf7c6e
 centralus e2e failed ★#7492 (merged) 2
e2e (2)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-<id> to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-<id> to exist; provider Microsoft.RedHatOpenShift
15:15:39 UTC · failed at e2e · test Customer should be able to retrieve cluster and nodepool status active versions · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x22a4be5cfc40>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-gc2cf577p6fs" to finish creating: GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534",
      "name": "1dec4678-5a13-4201-9600-24dcb5a26534",
      "status": "Failed",
      "startTime": "2026-10-14T16:17:52.259133125Z",
      "endTime": "2026-10-14T16:46:52.310381675Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v20261001-np-1-dnjw8-hs9xx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20261001-np-1-dnjw8-hs9xx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534
            correlation ID: b81493f7-b048-4704-b54a-5a69ef8ad3b5
            client request ID: 4dd74744-6a7e-48d1-9b1b-985d5238151a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x22a4be5cfc40>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-gc2cf577p6fs" to finish creating: GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534",
      "name": "1dec4678-5a13-4201-9600-24dcb5a26534",
      "status": "Failed",
      "startTime": "2026-10-14T16:17:52.259133125Z",
      "endTime": "2026-10-14T16:46:52.310381675Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v20261001-np-1-dnjw8-hs9xx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v20261001-np-1-dnjw8-hs9xx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-gc2cf577p6fs--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dnjw8-hs9xx to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0375552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1dec4678-5a13-4201-9600-24dcb5a26534
            correlation ID: b81493f7-b048-4704-b54a-5a69ef8ad3b5
            client request ID: 4dd74744-6a7e-48d1-9b1b-985d5238151a
occurred
There is no failure as the matcher passed to Eventually succeeded on its most recent iteration
15:15:39 UTC · failed at e2e · test Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.000s.
delegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance
There is no failure as the matcher passed to Eventually succeeded on its most recent iteration
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.000s.
delegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance
There is no failure as the matcher passed to Eventually succeeded on its most recent iteration
2h 15m 34s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 19b136f
 westus3 e2e failed ⚠#7500 (open) 2
e2e (2)
clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 "3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04="
15:15:22 UTC · failed at e2e · test Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.001s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x181b25f73380>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 "3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04="
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.001s.
ClusterIssuer should become Ready (ACME account registered)
Expected success, but got an error:
    <*errors.errorString | 0x181b25f73380>: 
    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 "3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04="
    {
        s: "clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"3vAWYLSpgMpdmMCKnkDG1+GD7QF9y2L9E9AOfsGsY04=\"",
    }
| take 101, Post "<url>": context deadline exceeded
15:15:22 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xef19c8be760>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T16:04:48.343783Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:04:48.343783Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0xef19c8be740>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:04:48.343783Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0xef19b80c410>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0xef19c8be720>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:04:48.343783Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0xef19d88c000>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xef19c8be760>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T16:04:48.343783Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:04:48.343783Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0xef19c8be740>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:04:48.343783Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0xef19b80c410>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0xef19c8be720>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6b2l97k9wstp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T16:04:48.343783Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0xef19d88c000>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 12m 36s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head caf95c8
 westus3 n/a passed #7531 (closed) 0 n/a
2h 6m 16s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 7a0f123 · merge b69b4de
 centralus n/a passed #7451 (merged) 0 n/a
2h 22m 4s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head dd38437 · merge dd38437
 centralus n/a passed #7503 (merged) 0 n/a
2h 12m 53s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 0ccb10c
 centralus n/a passed #7483 (open) 0 n/a
2h 9m 31s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head cf7a1e9 · merge cf7a1e9
 centralus e2e failed #7529 (merged) 1
e2e (1)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift
14:29:52 UTC · failed at e2e · test ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool "np-vtxdl7" for cluster "cluster-candidate-5-0-vtxdl7"
Unexpected error:
    <*fmt.wrapError | 0x23d38cc9d0c0>: 
    failed to create NodePool np-vtxdl7: failed waiting for nodepool="np-vtxdl7" for cluster "cluster-candidate-5-0-vtxdl7" in resourcegroup="rg-candidate-5-0-llmp2gg5r5wc" to finish creating: GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2",
      "name": "53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2",
      "status": "Failed",
      "startTime": "2026-10-14T15:40:21.526109364Z",
      "endTime": "2026-10-14T16:09:23.214850022Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2
            correlation ID: 1427370e-268a-4518-bc82-b69439593955
            client request ID: 074febab-8e11-4f6f-b6d6-be4d97f6fe52
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool "np-vtxdl7" for cluster "cluster-candidate-5-0-vtxdl7"
Unexpected error:
    <*fmt.wrapError | 0x23d38cc9d0c0>: 
    failed to create NodePool np-vtxdl7: failed waiting for nodepool="np-vtxdl7" for cluster "cluster-candidate-5-0-vtxdl7" in resourcegroup="rg-candidate-5-0-llmp2gg5r5wc" to finish creating: GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2",
      "name": "53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2",
      "status": "Failed",
      "startTime": "2026-10-14T15:40:21.526109364Z",
      "endTime": "2026-10-14T16:09:23.214850022Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/53b3d6e3-5baf-4a04-b762-6c77f3e9c4a2
            correlation ID: 1427370e-268a-4518-bc82-b69439593955
            client request ID: 074febab-8e11-4f6f-b6d6-be4d97f6fe52
occurred
2h 24m 57s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 7e6cd50
 westus3 alert, e2e failed #7428 (open) 3
e2e (1)
ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift
14:25:41 UTC · failed at e2e · test ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x50b30376378>: 
    resource group "rg-candidate-4-23-zgqgjh9b82nm": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-23-v7mn4s" in resourcegroup="rg-candidate-4-23-zgqgjh9b82nm" to finish deleting: GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe",
      "name": "90b34b3b-49e0-48b7-b175-fed3e7efcffe",
      "status": "Failed",
      "startTime": "2026-10-14T15:55:23.629401759Z",
      "endTime": "2026-10-14T16:19:25.512560555Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe
            correlation ID: 18b3d50f-1bf0-4dc5-95df-d87efa3be15d
            client request ID: eef778ec-0385-499a-a74c-a4ea9c9eb634
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x50b30376378>: 
    resource group "rg-candidate-4-23-zgqgjh9b82nm": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-23-v7mn4s" in resourcegroup="rg-candidate-4-23-zgqgjh9b82nm" to finish deleting: GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe",
      "name": "90b34b3b-49e0-48b7-b175-fed3e7efcffe",
      "status": "Failed",
      "startTime": "2026-10-14T15:55:23.629401759Z",
      "endTime": "2026-10-14T16:19:25.512560555Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9234816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90b34b3b-49e0-48b7-b175-fed3e7efcffe
            correlation ID: 18b3d50f-1bf0-4dc5-95df-d87efa3be15d
            client request ID: eef778ec-0385-499a-a74c-a4ea9c9eb634
occurred
alert (2)
alert [hcp] KubeDeploymentReplicasMismatch fired
14:25:41 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T16:31:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9234816-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.246:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9234816/providers/microsoft.monitor/accounts/hcps-j9234816", namespace="ocm-arohcpci01-2tel1ribjod37oeomufoootv8i3qle3j-m3d4q7m2k7e5r4w", pod="arohcp-monitor-kube-state-metrics-7988dc648c-hm6fs", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tel1ribjod37oeomufoootv8i3qle3j-m3d4q7m2k7e5r4w/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T16:31:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9234816-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.246:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9234816/providers/microsoft.monitor/accounts/hcps-j9234816", namespace="ocm-arohcpci01-2tel1ribjod37oeomufoootv8i3qle3j-m3d4q7m2k7e5r4w", pod="arohcp-monitor-kube-state-metrics-7988dc648c-hm6fs", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tel1ribjod37oeomufoootv8i3qle3j-m3d4q7m2k7e5r4w/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
alert [svc] BackendControllerQueueDepthHigh fired
14:25:41 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T15:42:08Z
  Ended:       2026-10-14T15:59:10Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j9234816-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
Firing 2:
  State:       Resolved
  Started:     2026-10-14T15:47:08Z
  Ended:       2026-10-14T15:59:10Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j9234816-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 18m 55s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head c171e40 · merge c171e40
 westus3 e2e failed #7519 (merged) 1
e2e (1)
DNS for route host <route-host> did not resolve: DNS for <route-host> did not resolve within <duration> (last error: lookup <host> on <dns-server>: dial udp <ip>: i/o timeout): context deadline exceeded
14:05:09 UTC · failed at e2e · test Customer should be able to retrieve cluster and nodepool status active versions · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x1fd36b60420>: 
    DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x1fd36b60400>{
            msg: "DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x1fd36b60420>: 
    DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x1fd36b60400>{
            msg: "DNS for agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mdh26.apps.aro.v20261001.d30u.j7447552.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
2h 13m 22s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 8af0205 · merge 8af0205
 westus3 alert failed ★#7528 (merged) 1
alert (1)
alert [svc] KubeJobFailed fired
14:05:01 UTC · failed at alert · test [aro-hcp-observability] [svc] alert KubeJobFailed does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T14:59:38Z
  Severity:    Sev3
  Labels:      alertname="KubeJobFailed", cluster="ci01-j4159872-svc", component="kubernetes-infrastructure", condition="true", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.26:8080", job="kube-state-metrics", job_name="hubble-generate-certs-55c4c87c", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4159872/providers/microsoft.monitor/accounts/services-j4159872", namespace="kube-system", pod="arohcp-monitor-kube-state-metrics-79b88f5699-mgfnh", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Job kube-system/hubble-generate-certs-55c4c87c failed to complete. Removing failed job after investigation should clear this alert.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T14:59:38Z
  Severity:    Sev3
  Labels:      alertname="KubeJobFailed", cluster="ci01-j4159872-svc", component="kubernetes-infrastructure", condition="true", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.26:8080", job="kube-state-metrics", job_name="hubble-generate-certs-55c4c87c", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4159872/providers/microsoft.monitor/accounts/services-j4159872", namespace="kube-system", pod="arohcp-monitor-kube-state-metrics-79b88f5699-mgfnh", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Job kube-system/hubble-generate-certs-55c4c87c failed to complete. Removing failed job after investigation should clear this alert.
2h 10m 14s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head b6c0794 · merge b6c0794
 centralus e2e failed ★#7524 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
14:04:58 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1f58f85617c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T14:50:56.5915854Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:50:56.5915854Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1f58f8561780>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:50:56.5915854Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1f58f8640e10>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1f58f8561760>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:50:56.5915854Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1f58fa5156e0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1f58f85617c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T14:50:56.5915854Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:50:56.5915854Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1f58f8561780>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:50:56.5915854Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1f58f8640e10>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1f58f8561760>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-j24c2xcb49pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:50:56.5915854Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1f58fa5156e0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 45m 34s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head f026aec · merge d13cfae
 centralus e2e failed ★#6746 (merged) 2
e2e (2)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift
13:25:18 UTC · failed at e2e · test Customer should be able to update node pool labels and taints · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0xf6a2f24c800>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-lpcs9lxsfkb6" to finish creating: GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6",
      "name": "7476d679-0544-4ea6-98bc-8369b57375b6",
      "status": "Failed",
      "startTime": "2026-10-14T14:43:09.62497089Z",
      "endTime": "2026-10-14T15:12:14.517634212Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6
            correlation ID: b2aadf48-cb5e-49df-828e-ed7c131078ce
            client request ID: a92178cd-8f13-4632-bde1-e16a3dc65cc2
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0xf6a2f24c800>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-lpcs9lxsfkb6" to finish creating: GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6",
      "name": "7476d679-0544-4ea6-98bc-8369b57375b6",
      "status": "Failed",
      "startTime": "2026-10-14T14:43:09.62497089Z",
      "endTime": "2026-10-14T15:12:14.517634212Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5913088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7476d679-0544-4ea6-98bc-8369b57375b6
            correlation ID: b2aadf48-cb5e-49df-828e-ed7c131078ce
            client request ID: a92178cd-8f13-4632-bde1-e16a3dc65cc2
occurred
| take 101, Post "<url>": context deadline exceeded
13:25:18 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xa2f1074500>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T14:31:58.8625817Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:31:58.8625817Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0xa2f1074480>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:31:58.8625817Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0xa2f02b66e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0xa2f1074460>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:31:58.8625817Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0xa2f1a7e7b0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xa2f1074500>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T14:31:58.8625817Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:31:58.8625817Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0xa2f1074480>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:31:58.8625817Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0xa2f02b66e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0xa2f1074460>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4qkvvrqq565z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T14:31:58.8625817Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0xa2f1a7e7b0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
57m 41s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 8af0205 · merge 8af0205
 n/a unknown failed #7528 (merged) 1 non-artifact-backed failure (no junit artifacts)
1h 6m 48s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 814133c
 n/a unknown failed #7498 (open) 1 non-artifact-backed failure (no junit artifacts)
1h 5m 35s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 664258c
 n/a unknown failed #7499 (open) 1 non-artifact-backed failure (no junit artifacts)
1h 6m 42s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 868faef
 n/a unknown failed #7497 (open) 1 non-artifact-backed failure (no junit artifacts)
3h 6m 8s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 68d8220 · merge 68d8220
 westus3 n/a passed #7516 (merged) 0 n/a
2h 24m 42s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 981742e
 centralus alert, e2e failed #7522 (open) 3
e2e (1)
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version <version> is Partial (want Completed), started <duration> ago; provider Microsoft.RedHatOpenShift
12:43:47 UTC · failed at e2e · test Customer should respect cluster-wide node limits with nodepool autoscaling · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit
Unexpected error:
    <*fmt.wrapError | 0x175f66e5a600>: 
    failed to create HCP cluster node-limit-cluster: failed waiting for cluster="node-limit-cluster" in resourcegroup="node-limit-test-8brvsr9whg7b" to finish creating: GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497",
      "name": "cba8c76e-06a0-4c2a-b3e0-d699a7abb497",
      "status": "Failed",
      "startTime": "2026-10-14T13:47:00.778542361Z",
      "endTime": "2026-10-14T14:16:06.716990034Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 17m32s ago"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497
            correlation ID: 113b689f-35da-4e51-8207-8a80d51d19ea
            client request ID: d9df4b89-ae88-4991-9a25-651fec75b816
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit
Unexpected error:
    <*fmt.wrapError | 0x175f66e5a600>: 
    failed to create HCP cluster node-limit-cluster: failed waiting for cluster="node-limit-cluster" in resourcegroup="node-limit-test-8brvsr9whg7b" to finish creating: GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497",
      "name": "cba8c76e-06a0-4c2a-b3e0-d699a7abb497",
      "status": "Failed",
      "startTime": "2026-10-14T13:47:00.778542361Z",
      "endTime": "2026-10-14T14:16:06.716990034Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.37 is Partial (want Completed), started 17m32s ago"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2105600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cba8c76e-06a0-4c2a-b3e0-d699a7abb497
            correlation ID: 113b689f-35da-4e51-8207-8a80d51d19ea
            client request ID: d9df4b89-ae88-4991-9a25-651fec75b816
occurred
alert (2)
alert [hcp] KubePodNotReady fired
12:43:47 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubePodNotReady does not fire · suite aro-hcp-tests
Show raw failure
alert fired 6 time(s) (3 unknown, 3 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T14:05:08Z
  Ended:       2026-10-14T14:16:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2105600-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tejjhusc5envac7crsmjjhhgio0a4aj-k5o0u8o1m0i8z8z", pod="etcd-1", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tejjhusc5envac7crsmjjhhgio0a4aj-k5o0u8o1m0i8z8z/etcd-1 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-14T14:05:09Z
  Ended:       2026-10-14T14:16:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2105600-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tejjc08ajbv33a72bs84k5higrqkmc1-private-kas", pod="etcd-2", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tejjc08ajbv33a72bs84k5higrqkmc1-private-kas/etcd-2 has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-14T14:05:09Z
  Ended:       2026-10-14T14:16:07Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2105600-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tejjl9d0k8tfj5mg7df7i9dvhecig15-d7v8f1i0h1a6r7x", pod="etcd-1", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tejjl9d0k8tfj5mg7df7i9dvhecig15-d7v8f1i0h1a6r7x/etcd-1 has been in a non-ready state for longer than 5 minutes.
alert [svc] BackendControllerQueueDepthHigh fired
12:43:47 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T14:02:08Z
  Ended:       2026-10-14T14:22:08Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j2105600-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 17m 33s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head de7b83c
 westus3 n/a passed #7486 (open) 0 n/a
37m 41s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 981742e
 westus3 provision failed #7522 (open) 1
provision (1)
ERROR CODE: DeploymentNotFound; provider Microsoft.ContainerService
12:03:18 UTC · failed at provision · test Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2) · suite templatize-pipeline
Show raw failure
time=2026-10-14T12:33:58.591Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description="Step cluster-create\n  Kind: Shell\n  Command: ./aks-cluster-create\n"
time=2026-10-14T12:35:01.493Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err="stamp 2: error running Shell Step, failed to execute shell command: {\"time\":\"2026-10-14T12:33:58.60113061Z\",\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5008768-mgmt-2\",\"clusterName\":\"ci01-j5008768-mgmt-2\",\"region\":\"westus3\"}\n{\"time\":\"2026-10-14T12:34:00.818434525Z\",\"level\":\"INFO\",\"msg\":\"configured pools\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5008768-mgmt-2\",\"clusterName\":\"ci01-j5008768-mgmt-2\",\"region\":\"westus3\",\"system\":1,\"worker\":3,\"infra\":2}\n{\"time\":\"2026-10-14T12:34:01.565464014Z\",\"level\":\"INFO\",\"msg\":\"deploying configured resources\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5008768-mgmt-2\",\"clusterName\":\"ci01-j5008768-mgmt-2\",\"region\":\"westus3\",\"deployment\":\"aks-cluster-create\",\"poolCount\":6,\"clusterFields\":null}\n{\"time\":\"2026-10-14T12:35:01.489201259Z\",\"level\":\"ERROR\",\"msg\":\"collecting deployment operations\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5008768-mgmt-2\",\"clusterName\":\"ci01-j5008768-mgmt-2\",\"region\":\"westus3\",\"deployment\":\"aks-cluster-create\",\"err\":{\"errorCode\":\"DeploymentNotFound\",\"statusCode\":404,\"errorMessage\":\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5008768-mgmt-2/deployments/aks-cluster-create/operations\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment 'aks-cluster-create' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"}}\n{\"time\":\"2026-10-14T12:35:01.489304211Z\",\"level\":\"ERROR\",\"msg\":\"run failed\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\",\"resourceGroup\":\"hcp-underlay-ci01-j5008768-mgmt-2\",\"clusterName\":\"ci01-j5008768-mgmt-2\",\"region\":\"westus3\",\"err\":\"submitting deployment aks-cluster-create: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5008768-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplateDeployment\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplateDeployment\\\",\\n    \\\"message\\\": \\\"The template deployment 'aks-cluster-create' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.ContainerService/managedClusters (2026-04-02-preview)' reported preflight validation errors. Tracking id is 'cd5c0f42-620b-41a9-b4be-4210990b303c'. See inner errors for details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"ListResourceSKUsFailed\\\",\\n        \\\"message\\\": \\\"Preflight validation check for resource(s) for container service ci01-j5008768-mgmt-2 in resource group hcp-underlay-ci01-j5008768-mgmt-2 failed. Message: List resource SKUs in subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX failed. Details: \\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\"elapsed\":\"1m2.888193483s\"}\n exit status 1"
2h 18m 38s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 43e42cb
 westus3 alert, e2e failed ⚠#7477 (closed) 12
e2e (11)
| take 101, Post "<url>": context deadline exceeded
12:02:45 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x23a2bb4ac840>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:53:15.9500124Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:53:15.9500124Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x23a2bb4ac820>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:53:15.9500124Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x23a2baa125f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x23a2bb4ac7c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:53:15.9500124Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x23a2bbe5b7d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x23a2bb4ac840>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:53:15.9500124Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:53:15.9500124Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x23a2bb4ac820>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:53:15.9500124Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x23a2baa125f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x23a2bb4ac7c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vqk49l2sww6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:53:15.9500124Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x23a2bbe5b7d0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
VerifyIngressControllerScope(External): timeout must be > 0, got 0s
12:02:45 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x362088f5cd60>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x362088f5cd60>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
timeout during CreateHCPClusterAndWait; context deadline exceeded
12:02:45 UTC · failed at e2e · test Customer should be able to create an HCP cluster with back-level version 4.20 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x14825bb062d0>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-gvb49lpzrwdc" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-gvb49lpzrwdc\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x14825b6451e0>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20
Unexpected error:
    <*fmt.wrapErrors | 0x14825bb062d0>: 
    failed waiting for cluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-gvb49lpzrwdc" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-gvb49lpzrwdc\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0x14825b6451e0>{
                s: "timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-gvb49lpzrwdc",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
12:02:45 UTC · failed at e2e · test Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-r5bgzg" with version 4.21.36 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x8ec62e3d3a0>: 
    failed to create HCP cluster cluster-zstream-4-21-r5bgzg: failed waiting for cluster="cluster-zstream-4-21-r5bgzg" in resourcegroup="rg-zstream-upgrade-4-21-w5nkcm4pm5dg" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45e95df0-b33f-4178-bb98-678a2c7c86be
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45e95df0-b33f-4178-bb98-678a2c7c86be",
      "name": "45e95df0-b33f-4178-bb98-678a2c7c86be",
      "status": "Failed",
      "startTime": "2026-10-14T12:54:00.543115746Z",
      "endTime": "2026-10-14T13:23:09.352219243Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45e95df0-b33f-4178-bb98-678a2c7c86be
            correlation ID: f5c613a3-b305-4d0c-9d84-e893e36cf56a
            client request ID: 5e30994c-2236-4ea7-8201-0dbafff97024
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-21-r5bgzg" with version 4.21.36 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x8ec62e3d3a0>: 
    failed to create HCP cluster cluster-zstream-4-21-r5bgzg: failed waiting for cluster="cluster-zstream-4-21-r5bgzg" in resourcegroup="rg-zstream-upgrade-4-21-w5nkcm4pm5dg" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45e95df0-b33f-4178-bb98-678a2c7c86be
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45e95df0-b33f-4178-bb98-678a2c7c86be",
      "name": "45e95df0-b33f-4178-bb98-678a2c7c86be",
      "status": "Failed",
      "startTime": "2026-10-14T12:54:00.543115746Z",
      "endTime": "2026-10-14T13:23:09.352219243Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45e95df0-b33f-4178-bb98-678a2c7c86be
            correlation ID: f5c613a3-b305-4d0c-9d84-e893e36cf56a
            client request ID: 5e30994c-2236-4ea7-8201-0dbafff97024
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
12:02:45 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x3d27d8e245e0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-55bg2dgcrg5l" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a82ad41-0a06-4822-84a1-1eb71ed05bcd
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a82ad41-0a06-4822-84a1-1eb71ed05bcd",
      "name": "3a82ad41-0a06-4822-84a1-1eb71ed05bcd",
      "status": "Failed",
      "startTime": "2026-10-14T12:54:01.175752572Z",
      "endTime": "2026-10-14T13:23:09.127667845Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a82ad41-0a06-4822-84a1-1eb71ed05bcd
            correlation ID: 37bfdd31-fb1e-43ba-9399-ecf6e9791352
            client request ID: 18725096-af8d-409d-b00b-a4455baf7cdf
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x3d27d8e245e0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-55bg2dgcrg5l" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a82ad41-0a06-4822-84a1-1eb71ed05bcd
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a82ad41-0a06-4822-84a1-1eb71ed05bcd",
      "name": "3a82ad41-0a06-4822-84a1-1eb71ed05bcd",
      "status": "Failed",
      "startTime": "2026-10-14T12:54:01.175752572Z",
      "endTime": "2026-10-14T13:23:09.127667845Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a82ad41-0a06-4822-84a1-1eb71ed05bcd
            correlation ID: 37bfdd31-fb1e-43ba-9399-ecf6e9791352
            client request ID: 18725096-af8d-409d-b00b-a4455baf7cdf
occurred
VerifyIngressControllerScope(External): timeout must be > 0, got 0s
12:02:45 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.0 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x5b6d94696d0>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x5b6d94696d0>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is "uninstalling"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift
12:02:45 UTC · failed at e2e · test Customer should be able to lifecycle and confirm external auth on a cluster · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x29ce27c860f0>: 
    resource group "ea-list-7knxqnrqn2r4": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="ea-listrj8r2m" in resourcegroup="ea-list-7knxqnrqn2r4" to finish deleting: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca",
      "name": "7d6add57-b9e0-49e9-82fe-b8f41bea59ca",
      "status": "Failed",
      "startTime": "2026-10-14T13:07:39.341755996Z",
      "endTime": "2026-10-14T13:31:39.648045292Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2teirqf5984h2i8hr97b4hmu155vmka1 still exists (deletion dispatched at 2026-10-14T13:07:39Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca
            correlation ID: a29d8857-e9bf-4a4f-bec4-d3e72ff8ea78
            client request ID: 11bba4e8-0934-463b-98c2-cdf775749155
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:
    <*errors.joinError | 0x29ce27c860f0>: 
    resource group "ea-list-7knxqnrqn2r4": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="ea-listrj8r2m" in resourcegroup="ea-list-7knxqnrqn2r4" to finish deleting: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca",
      "name": "7d6add57-b9e0-49e9-82fe-b8f41bea59ca",
      "status": "Failed",
      "startTime": "2026-10-14T13:07:39.341755996Z",
      "endTime": "2026-10-14T13:31:39.648045292Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2teirqf5984h2i8hr97b4hmu155vmka1 still exists (deletion dispatched at 2026-10-14T13:07:39Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d6add57-b9e0-49e9-82fe-b8f41bea59ca
            correlation ID: a29d8857-e9bf-4a4f-bec4-d3e72ff8ea78
            client request ID: 11bba4e8-0934-463b-98c2-cdf775749155
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
12:02:45 UTC · failed at e2e · test Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-52jt2v" with version 4.20.41 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x13f2c008c540>: 
    failed to create HCP cluster cluster-zstream-4-20-52jt2v: failed waiting for cluster="cluster-zstream-4-20-52jt2v" in resourcegroup="rg-zstream-upgrade-4-20-bjhwvwdlt4lp" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a49ccfd0-254d-4384-a27e-48ac2f48cd6a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a49ccfd0-254d-4384-a27e-48ac2f48cd6a",
      "name": "a49ccfd0-254d-4384-a27e-48ac2f48cd6a",
      "status": "Failed",
      "startTime": "2026-10-14T12:53:41.212151698Z",
      "endTime": "2026-10-14T13:22:49.1434114Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a49ccfd0-254d-4384-a27e-48ac2f48cd6a
            correlation ID: ade1110a-3260-430a-b809-94b9015a7859
            client request ID: d38a4887-fd6f-4a13-a272-b77968441299
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-20-52jt2v" with version 4.20.41 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x13f2c008c540>: 
    failed to create HCP cluster cluster-zstream-4-20-52jt2v: failed waiting for cluster="cluster-zstream-4-20-52jt2v" in resourcegroup="rg-zstream-upgrade-4-20-bjhwvwdlt4lp" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a49ccfd0-254d-4384-a27e-48ac2f48cd6a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a49ccfd0-254d-4384-a27e-48ac2f48cd6a",
      "name": "a49ccfd0-254d-4384-a27e-48ac2f48cd6a",
      "status": "Failed",
      "startTime": "2026-10-14T12:53:41.212151698Z",
      "endTime": "2026-10-14T13:22:49.1434114Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a49ccfd0-254d-4384-a27e-48ac2f48cd6a
            correlation ID: ade1110a-3260-430a-b809-94b9015a7859
            client request ID: d38a4887-fd6f-4a13-a272-b77968441299
occurred
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift
12:02:45 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x1e54e0c22fe0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-4brh2985txg7" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9435abe3-ba3e-41c8-967b-f93935401a81
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9435abe3-ba3e-41c8-967b-f93935401a81",
      "name": "9435abe3-ba3e-41c8-967b-f93935401a81",
      "status": "Failed",
      "startTime": "2026-10-14T12:54:00.088628604Z",
      "endTime": "2026-10-14T13:23:09.275455946Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9435abe3-ba3e-41c8-967b-f93935401a81
            correlation ID: c44d765a-42d3-4332-860c-e157cd4e1700
            client request ID: b101fed0-d4ba-40d2-8e74-a19436c964a4
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:152]: failed to create HCP cluster
Unexpected error:
    <*fmt.wrapError | 0x1e54e0c22fe0>: 
    failed to create HCP cluster ingress-cert-le: failed waiting for cluster="ingress-cert-le" in resourcegroup="ingress-cert-le-4brh2985txg7" to finish creating: GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9435abe3-ba3e-41c8-967b-f93935401a81
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9435abe3-ba3e-41c8-967b-f93935401a81",
      "name": "9435abe3-ba3e-41c8-967b-f93935401a81",
      "status": "Failed",
      "startTime": "2026-10-14T12:54:00.088628604Z",
      "endTime": "2026-10-14T13:23:09.275455946Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4694784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9435abe3-ba3e-41c8-967b-f93935401a81
            correlation ID: c44d765a-42d3-4332-860c-e157cd4e1700
            client request ID: b101fed0-d4ba-40d2-8e74-a19436c964a4
occurred
VerifyIngressControllerScope(External): timeout must be > 0, got 0s
12:02:45 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x26b7f5580860>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x26b7f5580860>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
VerifyIngressControllerScope(External): timeout must be > 0, got 0s
12:02:45 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.23 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x79d7b2a2d50>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:187]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*errors.errorString | 0x79d7b2a2d50>: 
    VerifyIngressControllerScope(External): timeout must be > 0, got 0s
    {
        s: "VerifyIngressControllerScope(External): timeout must be > 0, got 0s",
    }
alert (1)
alert [hcp] KubeDeploymentReplicasMismatch fired
12:02:45 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T13:48:39Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j4694784-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.131:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4694784/providers/microsoft.monitor/accounts/hcps-j4694784", namespace="ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m", pod="arohcp-monitor-kube-state-metrics-7988dc648c-gklrp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T13:48:39Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j4694784-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.131:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4694784/providers/microsoft.monitor/accounts/hcps-j4694784", namespace="ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m", pod="arohcp-monitor-kube-state-metrics-7988dc648c-gklrp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-14T13:48:39Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j4694784-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.131:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4694784/providers/microsoft.monitor/accounts/hcps-j4694784", namespace="ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m", pod="arohcp-monitor-kube-state-metrics-7988dc648c-gklrp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-14T13:48:40Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j4694784-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cluster-api", endpoint="http", environment="ci01", instance="10.128.64.131:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4694784/providers/microsoft.monitor/accounts/hcps-j4694784", namespace="ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m", pod="arohcp-monitor-kube-state-metrics-7988dc648c-gklrp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teirqf5984h2i8hr97b4hmu155vmka1-ea-listrj8r2m/cluster-api has not matched the expected number of replicas for longer than 30 minutes.
2h 13m 2s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 2859d7d
 centralus alert, e2e failed ⚠#7525 (closed) 4
e2e (3)
failed to create nodepool <nodepool>: failed waiting for nodepool="<nodepool>" for cluster "<cluster>" in resourcegroup="<resource-group>" to finish creating: GET <url>
11:52:51 UTC · failed at e2e · test Customer should be able to delete a nodepool · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x2a84e4eaf300>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-kmw59x46gl29" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c0d9fdd7-ab89-43fd-bb14-1369b1b457a6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c0d9fdd7-ab89-43fd-bb14-1369b1b457a6",
      "name": "c0d9fdd7-ab89-43fd-bb14-1369b1b457a6",
      "status": "Failed",
      "startTime": "2026-10-14T12:52:44.69049902Z",
      "endTime": "2026-10-14T13:21:50.180181739Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kmw59x46gl29--managed/providers/Microsoft.Compute/virtualMachines/t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kmw59x46gl29--managed/providers/Microsoft.Compute/virtualMachines/t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c0d9fdd7-ab89-43fd-bb14-1369b1b457a6
            correlation ID: df4065fe-b20a-424e-a40c-325e8f340d1d
            client request ID: 8a51c2cf-89b8-4cc4-96a9-004376aa27f0
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x2a84e4eaf300>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-kmw59x46gl29" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c0d9fdd7-ab89-43fd-bb14-1369b1b457a6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c0d9fdd7-ab89-43fd-bb14-1369b1b457a6",
      "name": "c0d9fdd7-ab89-43fd-bb14-1369b1b457a6",
      "status": "Failed",
      "startTime": "2026-10-14T12:52:44.69049902Z",
      "endTime": "2026-10-14T13:21:50.180181739Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kmw59x46gl29--managed/providers/Microsoft.Compute/virtualMachines/t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kmw59x46gl29--managed/providers/Microsoft.Compute/virtualMachines/t2p8x5c0u4g8u4j-np-delete-main-tbrj2-hkw4h to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c0d9fdd7-ab89-43fd-bb14-1369b1b457a6
            correlation ID: df4065fe-b20a-424e-a40c-325e8f340d1d
            client request ID: 8a51c2cf-89b8-4cc4-96a9-004376aa27f0
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-<id> to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-<id> to exist; provider Microsoft.RedHatOpenShift
11:52:51 UTC · failed at e2e · test Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x346ef5b24080>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-xn2t4zqk2qxz" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f",
      "name": "439dbd7f-6ac7-4b48-8533-a8815df6b20f",
      "status": "Failed",
      "startTime": "2026-10-14T12:55:20.912229682Z",
      "endTime": "2026-10-14T13:24:20.249835656Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-rpntb-stl8c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-rpntb-stl8c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f
            correlation ID: eefc14a4-21ec-420e-9b63-e8df580da42a
            client request ID: b9d8fbaf-fe74-4488-8491-9a8ef5ba8fd7
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x346ef5b24080>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-xn2t4zqk2qxz" to finish creating: GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f",
      "name": "439dbd7f-6ac7-4b48-8533-a8815df6b20f",
      "status": "Failed",
      "startTime": "2026-10-14T12:55:20.912229682Z",
      "endTime": "2026-10-14T13:24:20.249835656Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-rpntb-stl8c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-rpntb-stl8c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-xn2t4zqk2qxz--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-rpntb-stl8c to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6626944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/439dbd7f-6ac7-4b48-8533-a8815df6b20f
            correlation ID: eefc14a4-21ec-420e-9b63-e8df580da42a
            client request ID: b9d8fbaf-fe74-4488-8491-9a8ef5ba8fd7
occurred
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in <url> context deadline exceeded
11:52:51 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2620bace6480>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x2620bace6460>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2620bace6480>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-54nwd6zcc79j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x2620bace6460>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
alert (1)
alert [svc] LeaderElectionLeaseStale fired
11:52:51 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T13:50:51Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j6626944-mgmt-2", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.172:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6626944/providers/microsoft.monitor/accounts/services-j6626944", namespace="klusterlet-2teil4jlo5u4upah2m22fgadgh8tjl7r", pod="arohcp-monitor-kube-state-metrics-7988dc648c-fkd9k", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2teil4jlo5u4upah2m22fgadgh8tjl7r on cluster ci01-j6626944-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
2h 34m 38s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 19b136f
 westus3 e2e failed ★#7500 (open) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
11:40:23 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x26aa13102380>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:36:30.2125195Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:36:30.2125195Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x26aa13102360>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:36:30.2125195Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x26aa13b7a140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x26aa13102340>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:36:30.2125195Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x26aa1570b6b0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x26aa13102380>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:36:30.2125195Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:36:30.2125195Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x26aa13102360>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:36:30.2125195Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x26aa13b7a140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x26aa13102340>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-m6jg5g7z6k99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:36:30.2125195Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x26aa1570b6b0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 28m 7s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 5bf7c6e · merge 5bf7c6e
 centralus alert, e2e failed ★#7492 (merged) 2
e2e (1)
| take 101, Post "<url>": context deadline exceeded
11:40:22 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1c100d69e160>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:41:15.1639581Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:41:15.1639581Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1c100d69e140>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:41:15.1639581Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1c100c93e140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1c100d69e120>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:41:15.1639581Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1c100ec86120>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1c100d69e160>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:41:15.1639581Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:41:15.1639581Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1c100d69e140>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:41:15.1639581Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1c100c93e140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1c100d69e120>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5vcv29tnbhr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:41:15.1639581Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1c100ec86120>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
alert (1)
alert [hcp] KubeDeploymentReplicasMismatch fired
11:40:22 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T13:39:08Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6247424-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.161:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247424/providers/microsoft.monitor/accounts/hcps-j6247424", namespace="ocm-arohcpci01-2teilhc5mm4ic63dh34rtjjvq01opomp-g6w5r6o0x7p6g4p", pod="arohcp-monitor-kube-state-metrics-7988dc648c-nf5wp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teilhc5mm4ic63dh34rtjjvq01opomp-g6w5r6o0x7p6g4p/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T13:39:08Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6247424-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.161:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247424/providers/microsoft.monitor/accounts/hcps-j6247424", namespace="ocm-arohcpci01-2teilhc5mm4ic63dh34rtjjvq01opomp-g6w5r6o0x7p6g4p", pod="arohcp-monitor-kube-state-metrics-7988dc648c-nf5wp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teilhc5mm4ic63dh34rtjjvq01opomp-g6w5r6o0x7p6g4p/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
2h 20m 59s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head c171e40 · merge c171e40
 westus3 e2e failed #7519 (merged) 1
e2e (1)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift
11:39:53 UTC · failed at e2e · test Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42
Unexpected error:
    <*fmt.wrapError | 0x14d7438108a0>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-crqbjs" in resourcegroup="rg-np-version-upgrade-crqbjs-qjf2nvf5dbdv" to finish creating: GET https://rp.j1950464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43701430-60e7-4f74-b928-d5838bbad254
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43701430-60e7-4f74-b928-d5838bbad254",
      "name": "43701430-60e7-4f74-b928-d5838bbad254",
      "status": "Failed",
      "startTime": "2026-10-14T12:47:17.673973373Z",
      "endTime": "2026-10-14T13:16:19.646021375Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1950464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43701430-60e7-4f74-b928-d5838bbad254
            correlation ID: e9109fa5-d7a7-4040-b46c-da115a4555af
            client request ID: 5f396bcc-4d88-4f23-9b63-aaeebfe3ae99
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42
Unexpected error:
    <*fmt.wrapError | 0x14d7438108a0>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-crqbjs" in resourcegroup="rg-np-version-upgrade-crqbjs-qjf2nvf5dbdv" to finish creating: GET https://rp.j1950464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43701430-60e7-4f74-b928-d5838bbad254
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43701430-60e7-4f74-b928-d5838bbad254",
      "name": "43701430-60e7-4f74-b928-d5838bbad254",
      "status": "Failed",
      "startTime": "2026-10-14T12:47:17.673973373Z",
      "endTime": "2026-10-14T13:16:19.646021375Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1950464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43701430-60e7-4f74-b928-d5838bbad254
            correlation ID: e9109fa5-d7a7-4040-b46c-da115a4555af
            client request ID: 5f396bcc-4d88-4f23-9b63-aaeebfe3ae99
occurred
2h 33m 0s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 5fa1d19 · merge 4e38817
 centralus e2e failed ★#7502 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
11:38:17 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x9d7ab424ec0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:42:55.5344891Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:42:55.5344891Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x9d7ab424e80>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:42:55.5344891Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x9d7ab1ee3c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x9d7ab424e20>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:42:55.5344891Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x9d7ad295cb0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x9d7ab424ec0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T12:42:55.5344891Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:42:55.5344891Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x9d7ab424e80>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:42:55.5344891Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x9d7ab1ee3c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x9d7ab424e20>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kqggl97x5hwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T12:42:55.5344891Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x9d7ad295cb0>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 5m 8s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head b6c0794 · merge b6c0794
 centralus e2e failed ★#7524 (merged) 1
e2e (1)
NS lookup for aro
11:15:25 UTC · failed at e2e · test Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.001s.
delegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance
Expected success, but got an error:
    <*fmt.wrapError | 0x371ee5ea5c0>: 
    NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout
    {
        msg: "NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout",
        err: <*net.DNSError | 0x371ee500500>{
            UnwrapErr: <*net.OpError | 0x371ee5004b0>{
                Op: "dial",
                Net: "udp",
                Source: nil,
                Addr: nil,
                Err: <*net.DNSError | 0x371ee500460>{
                    UnwrapErr: <*net.timeoutError | 0x17059be0>{},
                    Err: "i/o timeout",
                    Name: "ns4-07.azure-dns.info.",
                    Server: "",
                    IsTimeout: true,
                    IsTemporary: true,
                    IsNotFound: false,
                },
            },
            Err: "dial udp: lookup ns4-07.azure-dns.info.: i/o timeout",
            Name: "aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud.",
            Server: "172.30.0.10:53",
            IsTimeout: true,
            IsTemporary: true,
            IsNotFound: false,
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:394]: Timed out after 90.001s.
delegation and authoritative NS answers should remain consistent for at least 30 seconds before issuance
Expected success, but got an error:
    <*fmt.wrapError | 0x371ee5ea5c0>: 
    NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout
    {
        msg: "NS lookup for aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud via ns4-07.azure-dns.info.: lookup aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud. on 172.30.0.10:53: dial udp: lookup ns4-07.azure-dns.info.: i/o timeout",
        err: <*net.DNSError | 0x371ee500500>{
            UnwrapErr: <*net.OpError | 0x371ee5004b0>{
                Op: "dial",
                Net: "udp",
                Source: nil,
                Addr: nil,
                Err: <*net.DNSError | 0x371ee500460>{
                    UnwrapErr: <*net.timeoutError | 0x17059be0>{},
                    Err: "i/o timeout",
                    Name: "ns4-07.azure-dns.info.",
                    Server: "",
                    IsTimeout: true,
                    IsTemporary: true,
                    IsNotFound: false,
                },
            },
            Err: "dial udp: lookup ns4-07.azure-dns.info.: i/o timeout",
            Name: "aro.ingress-cert-le.sexr.j4985344.hcp.osadev.cloud.",
            Server: "172.30.0.10:53",
            IsTimeout: true,
            IsTemporary: true,
            IsNotFound: false,
        },
    }
1h 30m 14s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 944b372
 westus3 unknown failed #7477 (closed) 1 non-artifact-backed failure (no junit artifacts)
2h 13m 14s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 678c15c · merge 678c15c
 centralus e2e failed #7430 (merged) 1
e2e (1)
DNS for route host <route-host> did not resolve: DNS for <route-host> did not resolve within <duration> (last error: lookup <host> on <dns-server>: dial udp <ip>: i/o timeout): context deadline exceeded
10:27:49 UTC · failed at e2e · test Customer should be able to retrieve cluster and nodepool status active versions · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x14a4f3922100>: 
    DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x14a4f39220e0>{
            msg: "DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0x14a4f3922100>: 
    DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x14a4f39220e0>{
            msg: "DNS for agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cfkmf.apps.aro.v20261001.113i.j5059456.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
2h 16m 1s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 868faef
 westus3 e2e failed ★#7497 (open) 2
e2e (2)
failed to create nodepool <nodepool>: failed waiting for nodepool="<nodepool>" for cluster "<cluster>" in resourcegroup="<resource-group>" to finish creating: GET <url>
09:44:59 UTC · failed at e2e · test Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x111b36865bc0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-z6jkddr7rndp" to finish creating: GET https://rp.j3109888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e",
      "name": "1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e",
      "status": "Failed",
      "startTime": "2026-10-14T10:52:20.627496048Z",
      "endTime": "2026-10-14T11:21:22.872991306Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-vl6zw-95b6v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6jkddr7rndp--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vl6zw-95b6v to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-vl6zw-95b6v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6jkddr7rndp--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vl6zw-95b6v to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3109888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e
            correlation ID: e1e29812-acc5-41cf-bfc0-c357f1f35016
            client request ID: 00853d2c-65b0-4134-bc84-b8514c7e1fc5
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x111b36865bc0>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-z6jkddr7rndp" to finish creating: GET https://rp.j3109888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e",
      "name": "1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e",
      "status": "Failed",
      "startTime": "2026-10-14T10:52:20.627496048Z",
      "endTime": "2026-10-14T11:21:22.872991306Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-vl6zw-95b6v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6jkddr7rndp--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vl6zw-95b6v to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine ingress-cert-le-np-1-vl6zw-95b6v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6jkddr7rndp--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vl6zw-95b6v to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3109888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bc2dd3d-99e3-4e13-b2ea-cac8a020b50e
            correlation ID: e1e29812-acc5-41cf-bfc0-c357f1f35016
            client request ID: 00853d2c-65b0-4134-bc84-b8514c7e1fc5
| take 101, Post "<url>": context deadline exceeded
09:44:59 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1c75b2d142e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T10:30:46.5406574Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T10:30:46.5406574Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1c75b2d142a0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T10:30:46.5406574Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1c75b1b661e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1c75b2d14220>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T10:30:46.5406574Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1c75b37c0150>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1c75b2d142e0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T10:30:46.5406574Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T10:30:46.5406574Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1c75b2d142a0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T10:30:46.5406574Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1c75b1b661e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1c75b2d14220>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69ltwbjp7hr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T10:30:46.5406574Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1c75b37c0150>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 15m 26s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 814133c
 westus3 e2e failed ⚠#7498 (open) 1
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in <url> context deadline exceeded
09:43:19 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x128374ba040>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x128374ba020>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x128374ba040>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p2q7sjlczt4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0x128374ba020>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
2h 16m 19s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 664258c
 centralus alert failed #7499 (open) 1
alert (1)
alert [svc] LeaderElectionLeaseStale fired
09:39:48 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T11:31:38Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j3791744-mgmt-2", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.65.17:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3791744/providers/microsoft.monitor/accounts/services-j3791744", namespace="klusterlet-2tegn8bj3up44dpb63ra7tfa7s7e28mf", pod="arohcp-monitor-kube-state-metrics-7988dc648c-th67p", prometheus="prometheus/prometheus", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2tegn8bj3up44dpb63ra7tfa7s7e28mf on cluster ci01-j3791744-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
2h 24m 23s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head eb662be · merge 8c3c716
 westus3 alert, e2e failed ⚠#7383 (merged) 2
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row
09:37:08 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1dded76126c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T11:13:43.5550944Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c", ActivityId:"0eb2baf5-e792-455b-9c86-a0654265ab17", SubActivityId:"0eb2baf5-e792-455b-9c86-a0654265ab17", ActivityType:"GW.Http.CallContext", ParentActivityId:"0eb2baf5-e792-455b-9c86-a0654265ab17", ActivityStack:"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x1dded76126a0>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x1dded776cc00>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-14T11:13:43.5550944Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c",
                        ActivityId: "0eb2baf5-e792-455b-9c86-a0654265ab17",
                        SubActivityId: "0eb2baf5-e792-455b-9c86-a0654265ab17",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "0eb2baf5-e792-455b-9c86-a0654265ab17",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1dded76126c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T11:13:43.5550944Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c", ActivityId:"0eb2baf5-e792-455b-9c86-a0654265ab17", SubActivityId:"0eb2baf5-e792-455b-9c86-a0654265ab17", ActivityType:"GW.Http.CallContext", ParentActivityId:"0eb2baf5-e792-455b-9c86-a0654265ab17", ActivityStack:"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tfgpxb4klb8k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x1dded76126a0>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:43.5550944Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c\", ActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", SubActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0eb2baf5-e792-455b-9c86-a0654265ab17\", ActivityStack:\"(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x1dded776cc00>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-14T11:13:43.5550944Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c",
                        ActivityId: "0eb2baf5-e792-455b-9c86-a0654265ab17",
                        SubActivityId: "0eb2baf5-e792-455b-9c86-a0654265ab17",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "0eb2baf5-e792-455b-9c86-a0654265ab17",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;458198cd-6878-4ea0-8879-a65dd48c7d9c ARID=0eb2baf5-e792-455b-9c86-a0654265ab17 > GW.Http.CallContext/0eb2baf5-e792-455b-9c86-a0654265ab17)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
alert (1)
alert [svc] BackendControllerQueueDepthHigh fired
09:37:08 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T10:49:39Z
  Ended:       2026-10-14T10:58:41Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j4582144-svc", component="backend", name="clusterresources", region="westus3", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 28m 36s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 981742e
 westus3 e2e failed ⚠#7522 (open) 1
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row
09:30:21 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x21de3cbea400>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T11:13:29.6322749Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736", ActivityId:"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd", SubActivityId:"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd", ActivityType:"GW.Http.CallContext", ParentActivityId:"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd", ActivityStack:"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x21de3ca9e560>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x21de3e2c5500>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-14T11:13:29.6322749Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736",
                        ActivityId: "9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd",
                        SubActivityId: "9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x21de3cbea400>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T11:13:29.6322749Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736", ActivityId:"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd", SubActivityId:"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd", ActivityType:"GW.Http.CallContext", ParentActivityId:"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd", ActivityStack:"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-fxq6rqgzrkpd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x21de3ca9e560>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T11:13:29.6322749Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736\", ActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", SubActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd\", ActivityStack:\"(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x21de3e2c5500>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-14T11:13:29.6322749Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736",
                        ActivityId: "9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd",
                        SubActivityId: "9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;a517dabe-2339-417d-83d4-3b27c7edb736 ARID=9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd > GW.Http.CallContext/9f47d8f2-d8b0-493f-b7da-5e950a8a5dbd)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
2h 21m 50s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head bf3311f · merge bf3311f
 westus3 n/a passed #7162 (merged) 0 n/a
2h 44m 23s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head dd7b09d
 centralus e2e failed ⚠#7523 (open) 1
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in <url> context deadline exceeded
09:25:32 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xee59296c020>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0xee59296c000>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0xee59296c020>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-h55twqdpvpzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
        err: <*fmt.wrapError | 0xee59296c000>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
2h 6m 26s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head d195c1d · merge a800194
 westus3 n/a passed #7489 (merged) 0 n/a
2h 22m 55s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 19b136f
 westus3 alert, e2e failed ⚠#7500 (open) 5
e2e (2)
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift
08:48:48 UTC · failed at e2e · test Authorized CIDRs Connectivity should allow API access only from authorized VM IP · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster "cidr-connectivity-test" with authorized CIDRs
Unexpected error:
    <*fmt.wrapError | 0x225a34843540>: 
    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-5rp2rtlm9fj9" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619",
      "name": "680a3372-b50b-4c21-a07b-0256c4145619",
      "status": "Failed",
      "startTime": "2026-10-14T09:49:54.805487688Z",
      "endTime": "2026-10-14T10:18:57.194996434Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619
            correlation ID: 5895723a-3ff0-431a-a927-5b6e194fc741
            client request ID: 350b7a37-c335-451c-b2e9-99a3b665afa1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster "cidr-connectivity-test" with authorized CIDRs
Unexpected error:
    <*fmt.wrapError | 0x225a34843540>: 
    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-5rp2rtlm9fj9" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619",
      "name": "680a3372-b50b-4c21-a07b-0256c4145619",
      "status": "Failed",
      "startTime": "2026-10-14T09:49:54.805487688Z",
      "endTime": "2026-10-14T10:18:57.194996434Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/680a3372-b50b-4c21-a07b-0256c4145619
            correlation ID: 5895723a-3ff0-431a-a927-5b6e194fc741
            client request ID: 350b7a37-c335-451c-b2e9-99a3b665afa1
occurred
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-<id> to exist Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-<id> to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-<id> to exist Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-<id> to exist; provider Microsoft.RedHatOpenShift
08:48:48 UTC · failed at e2e · test Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x2e793540e048>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-4b88pr2h6f62" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe",
      "name": "02548047-b3fd-49b8-8426-7df711bcbfbe",
      "status": "Failed",
      "startTime": "2026-10-14T10:01:14.415831901Z",
      "endTime": "2026-10-14T10:30:17.083278262Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine cilium-cluster-cilium-np-794ws-42z9w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\nMachine cilium-cluster-cilium-np-794ws-44qrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cluster-cilium-np-794ws-42z9w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\nMachine cilium-cluster-cilium-np-794ws-44qrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe
            correlation ID: d40a80e8-ffc2-4aca-bd50-98d7806c38e8
            client request ID: 84b1cf39-1ee3-4c6a-80ac-30f66febe153
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x2e793540e048>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-4b88pr2h6f62" to finish creating: GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe",
      "name": "02548047-b3fd-49b8-8426-7df711bcbfbe",
      "status": "Failed",
      "startTime": "2026-10-14T10:01:14.415831901Z",
      "endTime": "2026-10-14T10:30:17.083278262Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine cilium-cluster-cilium-np-794ws-42z9w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\nMachine cilium-cluster-cilium-np-794ws-44qrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cluster-cilium-np-794ws-42z9w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-42z9w to exist\nMachine cilium-cluster-cilium-np-794ws-44qrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4b88pr2h6f62--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-794ws-44qrb to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1691264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02548047-b3fd-49b8-8426-7df711bcbfbe
            correlation ID: d40a80e8-ffc2-4aca-bd50-98d7806c38e8
            client request ID: 84b1cf39-1ee3-4c6a-80ac-30f66febe153
occurred
alert (3)
alert [hcp] KubePodCrashLooping fired
08:48:48 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T10:12:41Z
  Ended:       2026-10-14T10:40:45Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j1691264-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.235:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264", namespace="ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="59d27077-4da5-493d-b6e6-b015a3abbc8a"
  Description: Pod ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-14T10:12:41Z
  Ended:       2026-10-14T10:41:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j1691264-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.235:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264", namespace="ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p", pod="etcd-0", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="59d27077-4da5-493d-b6e6-b015a3abbc8a"
  Description: Pod ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd-0 (etcd) is in waiting state (reason: "CrashLoopBackOff").
alert [hcp] KubeStatefulSetReplicasMismatch fired
08:48:48 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T10:09:45Z
  Ended:       2026-10-14T10:36:51Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j1691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.235:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264", namespace="ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p", pod="arohcp-monitor-kube-state-metrics-7988dc648c-9qldk", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd has not matched the expected number of replicas for longer than 15 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-14T10:09:45Z
  Ended:       2026-10-14T10:36:51Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j1691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.235:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264", namespace="ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p", pod="arohcp-monitor-kube-state-metrics-7988dc648c-9qldk", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/etcd has not matched the expected number of replicas for longer than 15 minutes.
alert [hcp] KubeDeploymentReplicasMismatch fired
08:48:48 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T10:24:42Z
  Ended:       2026-10-14T10:36:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j1691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.235:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264", namespace="ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p", pod="arohcp-monitor-kube-state-metrics-7988dc648c-9qldk", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-14T10:24:42Z
  Ended:       2026-10-14T10:37:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j1691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.235:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1691264/providers/microsoft.monitor/accounts/hcps-j1691264", namespace="ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p", pod="arohcp-monitor-kube-state-metrics-7988dc648c-9qldk", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teg4gob3eq3kct1j85ncs4avp9immt5-r0t7q6r8g7p8n9p/capi-provider has not matched the expected number of replicas for longer than 30 minutes.
2h 14m 19s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 7502b5b · merge 7502b5b
 centralus n/a passed #7521 (merged) 0 n/a
2h 17m 11s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 418edbb · merge 418edbb
 centralus e2e failed ★#7515 (merged) 1
e2e (1)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/<node> to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/<node> to exist; provider Microsoft.RedHatOpenShift
08:43:30 UTC · failed at e2e · test ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-7tqpsp" for cluster "cluster-hs-pre-7tqpsp"
Unexpected error:
    <*fmt.wrapError | 0x9fbbba82620>: 
    failed to create NodePool np-7tqpsp: failed waiting for nodepool="np-7tqpsp" for cluster "cluster-hs-pre-7tqpsp" in resourcegroup="rg-hs-pre--jlgwrtjbwxnf" to finish creating: GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6",
      "name": "bae4fd0e-d65b-410a-a9d7-cb38c5e195e6",
      "status": "Failed",
      "startTime": "2026-10-14T09:53:58.678670385Z",
      "endTime": "2026-10-14T10:23:01.968138297Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6
            correlation ID: fb25957f-8e08-4f05-8265-7dd58e255cf0
            client request ID: 4be80cba-579f-4cb7-8b46-0f96bfb70faa
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-7tqpsp" for cluster "cluster-hs-pre-7tqpsp"
Unexpected error:
    <*fmt.wrapError | 0x9fbbba82620>: 
    failed to create NodePool np-7tqpsp: failed waiting for nodepool="np-7tqpsp" for cluster "cluster-hs-pre-7tqpsp" in resourcegroup="rg-hs-pre--jlgwrtjbwxnf" to finish creating: GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6",
      "name": "bae4fd0e-d65b-410a-a9d7-cb38c5e195e6",
      "status": "Failed",
      "startTime": "2026-10-14T09:53:58.678670385Z",
      "endTime": "2026-10-14T10:23:01.968138297Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--jlgwrtjbwxnf-7tqpsp--managed/providers/Microsoft.Compute/virtualMachines/a0b7j6n3g2s4x1b-np-7tqpsp-rqshb-xb9b7 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4313856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bae4fd0e-d65b-410a-a9d7-cb38c5e195e6
            correlation ID: fb25957f-8e08-4f05-8265-7dd58e255cf0
            client request ID: 4be80cba-579f-4cb7-8b46-0f96bfb70faa
occurred
2h 31m 22s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 619a68d · merge 619a68d
 westus3 n/a passed #7496 (merged) 0 n/a
2h 27m 42s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 9dc0439
 centralus unknown failed #7477 (closed) 1 non-artifact-backed failure (no junit artifacts)
2h 23m 56s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 5bf7c6e · merge 5bf7c6e
 westus3 e2e failed #7492 (merged) 1
e2e (1)
DNS for route host <route-host> did not resolve: DNS for <route-host> did not resolve within <duration> (last error: lookup <host> on <dns-server>: dial udp <ip>: i/o timeout): context deadline exceeded
07:42:37 UTC · failed at e2e · test Customer should create a cluster using v20260901preview API and verify cluster health · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x4eb350769e0>: 
    DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x4eb350769c0>{
            msg: "DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster "v20260901"
Unexpected error:
    <*fmt.wrapError | 0x4eb350769e0>: 
    DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x4eb350769c0>{
            msg: "DNS for agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wbh7b.apps.aro.v20260901.v0z4.j6109568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
2h 17m 17s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 0d1a9ea
 westus3 alert failed #7441 (open) 1
alert (1)
alert [svc] LeaderElectionLeaseStale fired
07:30:12 UTC · failed at alert · test [aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T09:24:45Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j5330560-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.95:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5330560/providers/microsoft.monitor/accounts/services-j5330560", namespace="klusterlet-2teeronv8cck09r0avpqk909akckphl1", pod="arohcp-monitor-kube-state-metrics-7988dc648c-q2tbz", prometheus="prometheus/prometheus", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2teeronv8cck09r0avpqk909akckphl1 on cluster ci01-j5330560-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T09:28:47Z
  Severity:    Sev3
  Labels:      alertname="LeaderElectionLeaseStale", cluster="ci01-j5330560-mgmt-1", component="leader-election", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.95:8080", job="kube-state-metrics", lease="work-manager", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5330560/providers/microsoft.monitor/accounts/services-j5330560", namespace="klusterlet-2teerhop2ihg11pahsbnlrpia7d7l5pi", pod="arohcp-monitor-kube-state-metrics-7988dc648c-q2tbz", prometheus="prometheus/prometheus", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Leader election lease work-manager in namespace klusterlet-2teerhop2ihg11pahsbnlrpia7d7l5pi on cluster ci01-j5330560-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.
2h 14m 17s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head c4ce4c4 · merge d13cfae
 westus3 alert, e2e failed ★#6746 (merged) 2
e2e (1)
| take 101, Post "<url>": context deadline exceeded
07:21:49 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x234c4e8e85c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T08:11:01.1707477Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:11:01.1707477Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x234c4e8e8580>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:11:01.1707477Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x234c4e602370>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x234c4e8e8560>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:11:01.1707477Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x234c4ec6ec60>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x234c4e8e85c0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T08:11:01.1707477Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:11:01.1707477Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x234c4e8e8580>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:11:01.1707477Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x234c4e602370>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x234c4e8e8560>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99ktcjbpnm88/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:11:01.1707477Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x234c4ec6ec60>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
alert (1)
alert [hcp] KubeDeploymentReplicasMismatch fired
07:21:49 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T09:07:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j1186560-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.186:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1186560/providers/microsoft.monitor/accounts/hcps-j1186560", namespace="ocm-arohcpci01-2teemivtkpvljoobc4mgkttk1iligik4-m6t5k6r0u9k0a2k", pod="arohcp-monitor-kube-state-metrics-7988dc648c-klkpv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teemivtkpvljoobc4mgkttk1iligik4-m6t5k6r0u9k0a2k/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T09:07:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j1186560-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.186:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1186560/providers/microsoft.monitor/accounts/hcps-j1186560", namespace="ocm-arohcpci01-2teemivtkpvljoobc4mgkttk1iligik4-m6t5k6r0u9k0a2k", pod="arohcp-monitor-kube-state-metrics-7988dc648c-klkpv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teemivtkpvljoobc4mgkttk1iligik4-m6t5k6r0u9k0a2k/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
2h 15m 48s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 95cfd51 · merge 09587f6
 westus3 n/a passed #7440 (merged) 0 n/a
2h 12m 43s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head ca61586
 centralus e2e failed ★#7518 (closed) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
07:16:28 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x214cabccca20>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T08:10:50.6599172Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:10:50.6599172Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x214cabccc880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:10:50.6599172Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x214cabcb6f00>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x214cabccc840>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:10:50.6599172Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x214cae52d290>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x214cabccca20>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T08:10:50.6599172Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:10:50.6599172Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x214cabccc880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:10:50.6599172Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x214cabcb6f00>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x214cabccc840>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-kmmzkb92q7pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T08:10:50.6599172Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x214cae52d290>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 10m 4s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head d957f56 · merge b69b4de
 centralus e2e failed ★#7451 (merged) 2
e2e (2)
| take 101, Post "<url>": context deadline exceeded
07:11:07 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x303c35dfc8a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T07:57:15.3453027Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:57:15.3453027Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x303c35dfc880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:57:15.3453027Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x303c34e785f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x303c35dfc840>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:57:15.3453027Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x303c368c9800>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x303c35dfc8a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T07:57:15.3453027Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:57:15.3453027Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x303c35dfc880>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:57:15.3453027Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x303c34e785f0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x303c35dfc840>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qhx7w2fpqxp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:57:15.3453027Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x303c368c9800>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/<node> to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/<node> to exist; provider Microsoft.RedHatOpenShift
07:11:07 UTC · failed at e2e · test ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-cxf7bl" for cluster "cluster-hs-pre-cxf7bl"
Unexpected error:
    <*fmt.wrapError | 0x7e11cac4ba0>: 
    failed to create NodePool np-cxf7bl: failed waiting for nodepool="np-cxf7bl" for cluster "cluster-hs-pre-cxf7bl" in resourcegroup="rg-hs-pre--zccxz29md4tk" to finish creating: GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a",
      "name": "2a6b9db2-bba7-47f4-b565-c775480b382a",
      "status": "Failed",
      "startTime": "2026-10-14T08:08:22.741490309Z",
      "endTime": "2026-10-14T08:37:29.934217142Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a
            correlation ID: ca4a0e92-cce5-474f-a388-05ef9eae8408
            client request ID: 843e4749-4432-4dd5-a384-22418cffc290
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool "np-cxf7bl" for cluster "cluster-hs-pre-cxf7bl"
Unexpected error:
    <*fmt.wrapError | 0x7e11cac4ba0>: 
    failed to create NodePool np-cxf7bl: failed waiting for nodepool="np-cxf7bl" for cluster "cluster-hs-pre-cxf7bl" in resourcegroup="rg-hs-pre--zccxz29md4tk" to finish creating: GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a",
      "name": "2a6b9db2-bba7-47f4-b565-c775480b382a",
      "status": "Failed",
      "startTime": "2026-10-14T08:08:22.741490309Z",
      "endTime": "2026-10-14T08:37:29.934217142Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--zccxz29md4tk-cxf7bl--managed/providers/Microsoft.Compute/virtualMachines/d9f9j4p4h9j5g5y-np-cxf7bl-d29hq-jxkj6 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j1740672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2a6b9db2-bba7-47f4-b565-c775480b382a
            correlation ID: ca4a0e92-cce5-474f-a388-05ef9eae8408
            client request ID: 843e4749-4432-4dd5-a384-22418cffc290
occurred
2h 7m 39s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head c171e40 · merge c171e40
 centralus e2e failed ★#7519 (merged) 1
e2e (1)
| take 101, Post "<url>": context deadline exceeded
07:10:50 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x653b0686100>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T07:54:20.7143904Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:54:20.7143904Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x653b06860e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:54:20.7143904Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x653afe06140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x653b06860c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:54:20.7143904Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x653b1c88f60>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x653b0686100>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T07:54:20.7143904Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:54:20.7143904Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x653b06860e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:54:20.7143904Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x653afe06140>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x653b06860c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-d4whkkh9grgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T07:54:20.7143904Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x653b1c88f60>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 15m 8s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 68d8220 · merge 68d8220
 centralus alert, e2e failed ★#7516 (merged) 3
e2e (1)
VerifySwiftRouterChecks /subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row
07:02:43 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2a8ac5b76060>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T08:31:36.3290647Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4", ActivityId:"aec379bd-7447-4ac4-a50b-420184492383", SubActivityId:"aec379bd-7447-4ac4-a50b-420184492383", ActivityType:"GW.Http.CallContext", ParentActivityId:"aec379bd-7447-4ac4-a50b-420184492383", ActivityStack:"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x2a8ac5f001c0>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x2a8ac67f4100>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-14T08:31:36.3290647Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4",
                        ActivityId: "aec379bd-7447-4ac4-a50b-420184492383",
                        SubActivityId: "aec379bd-7447-4ac4-a50b-420184492383",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "aec379bd-7447-4ac4-a50b-420184492383",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2a8ac5b76060>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T08:31:36.3290647Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4", ActivityId:"aec379bd-7447-4ac4-a50b-420184492383", SubActivityId:"aec379bd-7447-4ac4-a50b-420184492383", ActivityType:"GW.Http.CallContext", ParentActivityId:"aec379bd-7447-4ac4-a50b-420184492383", ActivityStack:"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)"}, IsPermanent:false})
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tgr8grhvqs5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})",
        err: <*fmt.wrapError | 0x2a8ac5f001c0>{
            msg: "SWIFT query row: OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T08:31:36.3290647Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4\", ActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", SubActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"aec379bd-7447-4ac4-a50b-420184492383\", ActivityStack:\"(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)\"}, IsPermanent:false})",
            err: <*v2.OneApiError | 0x2a8ac67f4100>{
                ErrorMessage: {
                    Code: "LimitsExceeded",
                    Message: "Request is invalid and cannot be executed.",
                    Description: "Query execution has exceeded the allowed limits (80DA0001): 'join' operator has exceeded the memory budget (5368709120) during evaluation. Results may be incorrect or incomplete (E_RUNAWAY_QUERY; see https://aka.ms/kustoquerylimits)..",
                    Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                    Context: {
                        Timestamp: "2026-10-14T08:31:36.3290647Z",
                        ServiceAlias: "HCP-DEV-US-2",
                        MachineName: "",
                        ProcessName: "",
                        ProcessId: 0,
                        ThreadId: 0,
                        ClientRequestId: "KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4",
                        ActivityId: "aec379bd-7447-4ac4-a50b-420184492383",
                        SubActivityId: "aec379bd-7447-4ac4-a50b-420184492383",
                        ActivityType: "GW.Http.CallContext",
                        ParentActivityId: "aec379bd-7447-4ac4-a50b-420184492383",
                        ActivityStack: "(Activity stack: CRID=KGC.execute;3fe49d92-901b-4d5b-8acd-9d86ab8f1ed4 ARID=aec379bd-7447-4ac4-a50b-420184492383 > GW.Http.CallContext/aec379bd-7447-4ac4-a50b-420184492383)",
                    },
                    IsPermanent: false,
                },
            },
        },
    }
occurred
alert (2)
alert [hcp] KubeDeploymentReplicasMismatch fired
07:02:43 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T08:47:30Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0597632-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0597632/providers/microsoft.monitor/accounts/hcps-j0597632", namespace="ocm-arohcpci01-2teeckhc439vjfut8v9hg0n9h86opsdo-f6e4h0m8t0p5p0h", pod="arohcp-monitor-kube-state-metrics-7988dc648c-ksgxv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teeckhc439vjfut8v9hg0n9h86opsdo-f6e4h0m8t0p5p0h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T08:50:32Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0597632-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0597632/providers/microsoft.monitor/accounts/hcps-j0597632", namespace="ocm-arohcpci01-2teeckhc439vjfut8v9hg0n9h86opsdo-f6e4h0m8t0p5p0h", pod="arohcp-monitor-kube-state-metrics-7988dc648c-ksgxv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teeckhc439vjfut8v9hg0n9h86opsdo-f6e4h0m8t0p5p0h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-14T08:55:31Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0597632-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0597632/providers/microsoft.monitor/accounts/hcps-j0597632", namespace="ocm-arohcpci01-2teedfg7dgq7vc8sldn78hptbfgnuigo-k4g5z5l5z2j7p8j", pod="arohcp-monitor-kube-state-metrics-7988dc648c-ksgxv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teedfg7dgq7vc8sldn78hptbfgnuigo-k4g5z5l5z2j7p8j/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-14T08:55:31Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j0597632-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0597632/providers/microsoft.monitor/accounts/hcps-j0597632", namespace="ocm-arohcpci01-2teedfg7dgq7vc8sldn78hptbfgnuigo-k4g5z5l5z2j7p8j", pod="arohcp-monitor-kube-state-metrics-7988dc648c-ksgxv", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2teedfg7dgq7vc8sldn78hptbfgnuigo-k4g5z5l5z2j7p8j/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
alert [svc] BackendControllerQueueDepthHigh fired
07:02:43 UTC · failed at alert · test [aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T08:06:29Z
  Ended:       2026-10-14T08:26:30Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j0597632-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.
2h 15m 21s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 9bfd980
 westus3 e2e failed ⚠#6712 (open) 2
e2e (2)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist; provider Microsoft.RedHatOpenShift
06:47:51 UTC · failed at e2e · test Customer should be able to retrieve cluster and nodepool status active versions · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0xcf2616b6280>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-mfxbjq8jns8s" to finish creating: GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1",
      "name": "6ef2bbd7-ca39-48e4-90ab-26948b5b65f1",
      "status": "Failed",
      "startTime": "2026-10-14T07:49:18.1783953Z",
      "endTime": "2026-10-14T08:18:20.880236175Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine v20261001-np-1-cw6kh-fqbnw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\nMachine v20261001-np-1-cw6kh-qvbld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine v20261001-np-1-cw6kh-fqbnw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\nMachine v20261001-np-1-cw6kh-qvbld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1
            correlation ID: 39af48df-821f-41b4-9eda-383af9fbc767
            client request ID: d214e485-a699-4f3c-bb9e-2d4e1497eed9
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool "np-1" for v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0xcf2616b6280>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "v20261001" in resourcegroup="v20261001-mfxbjq8jns8s" to finish creating: GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1",
      "name": "6ef2bbd7-ca39-48e4-90ab-26948b5b65f1",
      "status": "Failed",
      "startTime": "2026-10-14T07:49:18.1783953Z",
      "endTime": "2026-10-14T08:18:20.880236175Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine v20261001-np-1-cw6kh-fqbnw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\nMachine v20261001-np-1-cw6kh-qvbld: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine v20261001-np-1-cw6kh-fqbnw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-fqbnw to exist\nMachine v20261001-np-1-cw6kh-qvbld: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001-mfxbjq8jns8s--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-cw6kh-qvbld to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9450880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ef2bbd7-ca39-48e4-90ab-26948b5b65f1
            correlation ID: 39af48df-821f-41b4-9eda-383af9fbc767
            client request ID: d214e485-a699-4f3c-bb9e-2d4e1497eed9
occurred
failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs
06:47:51 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.098s.
Expected success, but got an error:
    <*fmt.wrapError | 0x3618a9570d00>: 
    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs
    | where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'
    | distinct cid, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": net/http: TLS handshake timeout
    {
        msg: "failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
        err: <*fmt.wrapError | 0x3618a9570cc0>{
            msg: "failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
            err: <*fmt.wrapError | 0x3618a9570c60>{
                msg: "failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
                err: <*fmt.wrapError | 0x3618a9570c40>{
                    msg: "failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
                    err: <*errors.Error | 0x3618a9654280>{
                        Op: 1,
                        Kind: 7,
                        Err: <*fmt.wrapError | 0x3618a9570c00>{
                            msg: "With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
                            err: <*url.Error | 0x3618aa1308a0>{
                                Op: "Post",
                                URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                                Err: <http.tlsHandshakeTimeoutError>{},
                            },
                        },
                        restErrMsg: nil,
                        decoded: nil,
                        permanent: false,
                        inner: nil,
                    },
                },
            },
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 602.098s.
Expected success, but got an error:
    <*fmt.wrapError | 0x3618a9570d00>: 
    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs
    | where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'
    | distinct cid, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": net/http: TLS handshake timeout
    {
        msg: "failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
        err: <*fmt.wrapError | 0x3618a9570cc0>{
            msg: "failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
            err: <*fmt.wrapError | 0x3618a9570c60>{
                msg: "failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
                err: <*fmt.wrapError | 0x3618a9570c40>{
                    msg: "failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
                    err: <*errors.Error | 0x3618a9654280>{
                        Op: 1,
                        Kind: 7,
                        Err: <*fmt.wrapError | 0x3618a9570c00>{
                            msg: "With query: clustersServiceLogs\n| where resource_id has '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5967pdcp8vw8'\n| distinct cid, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": net/http: TLS handshake timeout",
                            err: <*url.Error | 0x3618aa1308a0>{
                                Op: "Post",
                                URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                                Err: <http.tlsHandshakeTimeoutError>{},
                            },
                        },
                        restErrMsg: nil,
                        decoded: nil,
                        permanent: false,
                        inner: nil,
                    },
                },
            },
        },
    }
2h 11m 53s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head bb4f900 · merge bb4f900
 centralus e2e failed ★#7509 (merged) 1
e2e (1)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-<id> to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-<id> to exist; provider Microsoft.RedHatOpenShift
06:41:43 UTC · failed at e2e · test Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x3a53ed53e100>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-b6z74shd5q4f" to finish creating: GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8",
      "name": "e37b257b-a800-4d32-9117-6f5dadd22fa8",
      "status": "Failed",
      "startTime": "2026-10-14T07:43:11.635376265Z",
      "endTime": "2026-10-14T08:12:15.916382682Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-nzqkm-w8mpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-nzqkm-w8mpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8
            correlation ID: a458a580-f3b1-4361-8ade-4eb50ff689a4
            client request ID: ac6a0de7-84cd-4baa-8bf7-d9a0e674cba5
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool "np-1" for private ingress cluster "private-ingress"
Unexpected error:
    <*fmt.wrapError | 0x3a53ed53e100>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "private-ingress" in resourcegroup="private-ingress-b6z74shd5q4f" to finish creating: GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8",
      "name": "e37b257b-a800-4d32-9117-6f5dadd22fa8",
      "status": "Failed",
      "startTime": "2026-10-14T07:43:11.635376265Z",
      "endTime": "2026-10-14T08:12:15.916382682Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine private-ingress-np-1-nzqkm-w8mpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine private-ingress-np-1-nzqkm-w8mpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b6z74shd5q4f--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-nzqkm-w8mpn to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0292224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e37b257b-a800-4d32-9117-6f5dadd22fa8
            correlation ID: a458a580-f3b1-4361-8ade-4eb50ff689a4
            client request ID: ac6a0de7-84cd-4baa-8bf7-d9a0e674cba5
occurred
2h 15m 42s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head d1ae0b6 · merge d1ae0b6
 westus3 n/a passed #7517 (merged) 0 n/a
2h 21m 14s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 418edbb · merge 418edbb
 centralus e2e failed #7515 (merged) 1
e2e (1)
DNS for route host <route-host> did not resolve: DNS for <route-host> did not resolve within <duration> (last error: lookup <host> on <dns-server>: dial udp <ip>: i/o timeout): context deadline exceeded
06:17:37 UTC · failed at e2e · test ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster "cluster-hs-pre-mntlmg"
Unexpected error:
    <*fmt.wrapError | 0x116df72e380>: 
    DNS for route host agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x116df72e360>{
            msg: "DNS for agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster "cluster-hs-pre-mntlmg"
Unexpected error:
    <*fmt.wrapError | 0x116df72e380>: 
    DNS for route host agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x116df72e360>{
            msg: "DNS for agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wn5wg.apps.aro.h7s4t6m2m3n4t3w.l7ob.j2225408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
18m 39s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 10af1db · merge 68d8220
 n/a unknown failed #7516 (merged) 1 non-artifact-backed failure (no junit artifacts)
2h 23m 34s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head ff90e86
 centralus alert, e2e failed #7504 (open) 2
e2e (1)
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version <version> is Partial (want Completed), started <duration> ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift
06:10:15 UTC · failed at e2e · test ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fzqsptw7bpr/cluster-candidate-5-0-g7bpwx should provision
Unexpected error:
    <*fmt.wrapError | 0x4abdbffea60>: 
    failed to create HCP cluster cluster-candidate-5-0-g7bpwx: failed waiting for cluster="cluster-candidate-5-0-g7bpwx" in resourcegroup="rg-candidate-5-0-8fzqsptw7bpr" to finish creating: GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454",
      "name": "b237387d-d51f-41fb-8762-715d9d2b7454",
      "status": "Failed",
      "startTime": "2026-10-14T07:05:39.437908469Z",
      "endTime": "2026-10-14T07:34:48.061766511Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.6 is Partial (want Completed), started 12m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454
            correlation ID: 92af7c84-e19d-482a-a1fe-a87a2b603856
            client request ID: 2dc5545b-5fc8-4471-80a2-89372581489c
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fzqsptw7bpr/cluster-candidate-5-0-g7bpwx should provision
Unexpected error:
    <*fmt.wrapError | 0x4abdbffea60>: 
    failed to create HCP cluster cluster-candidate-5-0-g7bpwx: failed waiting for cluster="cluster-candidate-5-0-g7bpwx" in resourcegroup="rg-candidate-5-0-8fzqsptw7bpr" to finish creating: GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454",
      "name": "b237387d-d51f-41fb-8762-715d9d2b7454",
      "status": "Failed",
      "startTime": "2026-10-14T07:05:39.437908469Z",
      "endTime": "2026-10-14T07:34:48.061766511Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.6 is Partial (want Completed), started 12m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3579520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b237387d-d51f-41fb-8762-715d9d2b7454
            correlation ID: 92af7c84-e19d-482a-a1fe-a87a2b603856
            client request ID: 2dc5545b-5fc8-4471-80a2-89372581489c
occurred
alert (1)
alert [hcp] KubePodNotReady fired
06:10:15 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubePodNotReady does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T07:29:03Z
  Ended:       2026-10-14T07:43:03Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3579520-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tednh1ctcvl912rk2fj6486rd5emipo-j8c3b0w9t8v9t1z", pod="router-6fb85f79bd-67zsz", region="centralus", severity="warning"
  Description: Pod ocm-arohcpci01-2tednh1ctcvl912rk2fj6486rd5emipo-j8c3b0w9t8v9t1z/router-6fb85f79bd-67zsz has been in a non-ready state for longer than 5 minutes.
1h 58m 7s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 437c087
 centralus e2e failed ⚠#7477 (closed) 3
e2e (3)
VerifyIngressControllerScope(External) timed out after 5m0
05:04:24 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*fmt.wrapError | 0x1ba452ab87c0>: 
    VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded
    {
        msg: "VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
        err: <*fmt.wrapError | 0x1ba452ab87a0>{
            msg: "failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
            err: <*fmt.wrapError | 0x1ba452ab8760>{
                msg: "client rate limiter Wait returned an error: context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*fmt.wrapError | 0x1ba452ab87c0>: 
    VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded
    {
        msg: "VerifyIngressControllerScope(External) timed out after 5m0.006s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
        err: <*fmt.wrapError | 0x1ba452ab87a0>{
            msg: "failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
            err: <*fmt.wrapError | 0x1ba452ab8760>{
                msg: "client rate limiter Wait returned an error: context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
| take 101, Post "<url>": context deadline exceeded
05:04:24 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x279f2da40c00>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T05:46:55.1969919Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x279f2da40be0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x279f2c9fa820>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x279f2da40bc0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x279f2eb19620>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x279f2da40c00>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T05:46:55.1969919Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x279f2da40be0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x279f2c9fa820>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x279f2da40bc0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdmzjxjc7lck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T05:46:55.1969919Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x279f2eb19620>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
VerifyIngressControllerScope(External) timed out after 5m0
05:04:24 UTC · failed at e2e · test ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*fmt.wrapError | 0x2c6abc8c6320>: 
    VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded
    {
        msg: "VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
        err: <*fmt.wrapError | 0x2c6abc8c62a0>{
            msg: "failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
            err: <*fmt.wrapError | 0x2c6abc8c6260>{
                msg: "client rate limiter Wait returned an error: context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:199]: default ingress certificate scenario requires public ingress
Expected success, but got an error:
    <*fmt.wrapError | 0x2c6abc8c6320>: 
    VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded
    {
        msg: "VerifyIngressControllerScope(External) timed out after 5m0.002s: failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
        err: <*fmt.wrapError | 0x2c6abc8c62a0>{
            msg: "failed to get default IngressController: client rate limiter Wait returned an error: context deadline exceeded",
            err: <*fmt.wrapError | 0x2c6abc8c6260>{
                msg: "client rate limiter Wait returned an error: context deadline exceeded",
                err: <context.deadlineExceededError>{},
            },
        },
    }
2h 17m 48s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head eda29a9 · merge eda29a9
 westus3 n/a passed #7510 (merged) 0 n/a
2h 10m 11s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 9bfd980
 westus3 alert failed #6712 (open) 1
alert (1)
alert [hcp] KubeDeploymentReplicasMismatch fired
04:11:31 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-14T05:54:57Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9250048-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9250048/providers/microsoft.monitor/accounts/hcps-j9250048", namespace="ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress", pod="arohcp-monitor-kube-state-metrics-7988dc648c-ffckj", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-14T05:54:57Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9250048-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="cloud-network-config-controller", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9250048/providers/microsoft.monitor/accounts/hcps-j9250048", namespace="ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress", pod="arohcp-monitor-kube-state-metrics-7988dc648c-ffckj", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tebrqnbao74dvqvkb08nka1b7h6q9t1-private-ingress/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.
2h 28m 17s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head ff90e86
 westus3 e2e failed ★#7504 (open) 2
e2e (2)
cluster <resource-group>/<cluster> entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
03:39:51 UTC · failed at e2e · test ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-pljzvnf827th/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-pljzvnf827th/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
| take 101, Post "<url>": context deadline exceeded
03:39:51 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2624930243a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T04:36:17.5028906Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x262493024380>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2624924923c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x262493024340>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x26249412ae10>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x2624930243a0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T04:36:17.5028906Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x262493024380>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x2624924923c0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x262493024340>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-pdr4v26jhp4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T04:36:17.5028906Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x26249412ae10>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
2h 17m 43s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 29ee7d9 · merge 29ee7d9
 centralus n/a passed #7248 (merged) 0 n/a
2h 24m 37s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 9259901 · merge 9259901
 westus3 n/a passed #7512 (merged) 0 n/a
2h 22m 51s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head bb4f900 · merge bb4f900
 centralus e2e failed #7509 (merged) 1
e2e (1)
ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift
03:39:38 UTC · failed at e2e · test Customer should be able to use workload identity via the cluster OIDC issuer URL · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster
Unexpected error:
    <*fmt.wrapError | 0x3e322e6e67c0>: 
    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster="oidc-wi-cluster" in resourcegroup="oidc-wi-7rswnntg2mwm" to finish creating: GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71",
      "name": "eeb29541-e4cc-419e-8a9e-17c262940b71",
      "status": "Failed",
      "startTime": "2026-10-14T05:02:26.935820049Z",
      "endTime": "2026-10-14T05:31:33.678218143Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71
            correlation ID: 91b1a5de-a9bc-4e6e-ad6d-e45a7cb1831a
            client request ID: 4a6ae7e3-e39d-4cda-bb39-3082e7e48f98
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster
Unexpected error:
    <*fmt.wrapError | 0x3e322e6e67c0>: 
    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster="oidc-wi-cluster" in resourcegroup="oidc-wi-7rswnntg2mwm" to finish creating: GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71",
      "name": "eeb29541-e4cc-419e-8a9e-17c262940b71",
      "status": "Failed",
      "startTime": "2026-10-14T05:02:26.935820049Z",
      "endTime": "2026-10-14T05:31:33.678218143Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j2666368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eeb29541-e4cc-419e-8a9e-17c262940b71
            correlation ID: 91b1a5de-a9bc-4e6e-ad6d-e45a7cb1831a
            client request ID: 4a6ae7e3-e39d-4cda-bb39-3082e7e48f98
occurred
2h 19m 17s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head 634ea63 · merge efd4ab7
 westus3 n/a passed #7305 (merged) 0 n/a
2h 12m 11s pull-ci-Azure-ARO-HCP-main-e2e-parallel
head 3c936e1
 westus3 alert failed ⚠#7495 (open) 3
alert (3)
alert [hcp] KubePodNotReady fired
02:41:42 UTC · failed at alert · test [aro-hcp-observability] [hcp] alert KubePodNotReady does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T04:04:44Z
  Ended:       2026-10-14T04:10:41Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j7369856-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2teajl722c52pqdr760m7glllbuaiq3i-u9l0s4v1d6h4w4k", pod="cluster-node-tuning-operator-576b745fcc-qpnjx", region="westus3", severity="warning"
  Description: Pod ocm-arohcpci01-2teajl722c52pqdr760m7glllbuaiq3i-u9l0s4v1d6h4w4k/cluster-node-tuning-operator-576b745fcc-qpnjx has been in a non-ready state for longer than 5 minutes.
alert [infra] Cosmos DB Normalized RU Consumption High - <cosmos-account> fired
02:41:42 UTC · failed at alert · test [aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7369856 does not fire · suite aro-hcp-tests
Show raw failure
alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T03:48:00Z
  Ended:       2026-10-14T04:18:05Z
  Severity:    Sev3
  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units
alert [infra] Cosmos DB Normalized RU Consumption Burst High - <cosmos-account> fired
02:41:42 UTC · failed at alert · test [aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j7369856 does not fire · suite aro-hcp-tests
Show raw failure
alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-14T03:40:18Z
  Ended:       2026-10-14T03:43:19Z
  Severity:    Sev3
  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units
Firing 2:
  State:       Resolved
  Started:     2026-10-14T03:52:13Z
  Ended:       2026-10-14T04:06:24Z
  Severity:    Sev3
  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units
2h 27m 55s pull-ci-Azure-ARO-HCP-main-e2e-parallel
merged PR
head b8a38dc · merge 8d693ac
 westus3 e2e failed #6968 (merged) 6
e2e (6)
ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation "hypershift.openshift.io/cluster-size-override" is "e2e_minimal; provider Microsoft.RedHatOpenShift
02:08:42 UTC · failed at e2e · test Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-5-1-tlhwld" to 5.1
Unexpected error:
    <*fmt.wrapError | 0x2c0311788240>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-5-1-tlhwld" in resourcegroup="rg-cp-ystream-upgrade-5-1-fgzfp7lmmmc9" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc0f8306-6fb7-4fdc-b059-86e5b94c7910
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc0f8306-6fb7-4fdc-b059-86e5b94c7910",
      "name": "cc0f8306-6fb7-4fdc-b059-86e5b94c7910",
      "status": "Failed",
      "startTime": "2026-10-14T03:27:16.482132272Z",
      "endTime": "2026-10-14T04:11:24.926195758Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc0f8306-6fb7-4fdc-b059-86e5b94c7910
            correlation ID: fdec914a-b634-4ccd-a97c-27f1baa6c4c6
            client request ID: 6c1f8e2d-9bab-4951-9d1c-7f16027e6297
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-5-1-tlhwld" to 5.1
Unexpected error:
    <*fmt.wrapError | 0x2c0311788240>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-5-1-tlhwld" in resourcegroup="rg-cp-ystream-upgrade-5-1-fgzfp7lmmmc9" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc0f8306-6fb7-4fdc-b059-86e5b94c7910
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc0f8306-6fb7-4fdc-b059-86e5b94c7910",
      "name": "cc0f8306-6fb7-4fdc-b059-86e5b94c7910",
      "status": "Failed",
      "startTime": "2026-10-14T03:27:16.482132272Z",
      "endTime": "2026-10-14T04:11:24.926195758Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc0f8306-6fb7-4fdc-b059-86e5b94c7910
            correlation ID: fdec914a-b634-4ccd-a97c-27f1baa6c4c6
            client request ID: 6c1f8e2d-9bab-4951-9d1c-7f16027e6297
occurred
ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation "hypershift.openshift.io/cluster-size-override" is "e2e_minimal; provider Microsoft.RedHatOpenShift
02:08:42 UTC · failed at e2e · test Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-4-21-fwj9dh" to 4.21
Unexpected error:
    <*fmt.wrapError | 0x34cefee33220>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-4-21-fwj9dh" in resourcegroup="rg-cp-ystream-upgrade-4-21-8xh7kcthzdjn" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d579950b-9381-41a7-88e3-054e6757d7f3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d579950b-9381-41a7-88e3-054e6757d7f3",
      "name": "d579950b-9381-41a7-88e3-054e6757d7f3",
      "status": "Failed",
      "startTime": "2026-10-14T03:16:20.176669968Z",
      "endTime": "2026-10-14T04:00:20.21258051Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d579950b-9381-41a7-88e3-054e6757d7f3
            correlation ID: d71e6775-e515-48ff-85d4-84e59e0b5fa2
            client request ID: 1f8532bb-fb76-4062-a893-c1c94849ca49
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-4-21-fwj9dh" to 4.21
Unexpected error:
    <*fmt.wrapError | 0x34cefee33220>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-4-21-fwj9dh" in resourcegroup="rg-cp-ystream-upgrade-4-21-8xh7kcthzdjn" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d579950b-9381-41a7-88e3-054e6757d7f3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d579950b-9381-41a7-88e3-054e6757d7f3",
      "name": "d579950b-9381-41a7-88e3-054e6757d7f3",
      "status": "Failed",
      "startTime": "2026-10-14T03:16:20.176669968Z",
      "endTime": "2026-10-14T04:00:20.21258051Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d579950b-9381-41a7-88e3-054e6757d7f3
            correlation ID: d71e6775-e515-48ff-85d4-84e59e0b5fa2
            client request ID: 1f8532bb-fb76-4062-a893-c1c94849ca49
occurred
ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation "hypershift.openshift.io/cluster-size-override" is "e2e_minimal; provider Microsoft.RedHatOpenShift
02:08:42 UTC · failed at e2e · test Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-4-22-8fmvrw" to 4.22
Unexpected error:
    <*fmt.wrapError | 0x189d56536760>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-4-22-8fmvrw" in resourcegroup="rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912",
      "name": "008de110-0fc7-4cfb-b1a8-16dd774cc912",
      "status": "Failed",
      "startTime": "2026-10-14T03:16:19.986510176Z",
      "endTime": "2026-10-14T04:00:20.242906036Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912
            correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21
            client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-4-22-8fmvrw" to 4.22
Unexpected error:
    <*fmt.wrapError | 0x189d56536760>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-4-22-8fmvrw" in resourcegroup="rg-cp-ystream-upgrade-4-22-m4rkshgrgs9x" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912",
      "name": "008de110-0fc7-4cfb-b1a8-16dd774cc912",
      "status": "Failed",
      "startTime": "2026-10-14T03:16:19.986510176Z",
      "endTime": "2026-10-14T04:00:20.242906036Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/008de110-0fc7-4cfb-b1a8-16dd774cc912
            correlation ID: edcafa24-51c5-48f1-9ce5-cdc56c9cfe21
            client request ID: d7ffd994-e6d8-430b-b52c-c25abf00d2d4
occurred
ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation "hypershift.openshift.io/cluster-size-override" is "e2e_minimal; provider Microsoft.RedHatOpenShift
02:08:42 UTC · failed at e2e · test Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-4-23-trt4r8" to 4.23
Unexpected error:
    <*fmt.wrapError | 0xbc2497a0a40>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-4-23-trt4r8" in resourcegroup="rg-cp-ystream-upgrade-4-23-nm455m5dz4rg" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3",
      "name": "ab2c099b-e08d-48bf-8615-536073622db3",
      "status": "Failed",
      "startTime": "2026-10-14T03:17:19.784745543Z",
      "endTime": "2026-10-14T04:01:20.236478176Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3
            correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145
            client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-4-23-trt4r8" to 4.23
Unexpected error:
    <*fmt.wrapError | 0xbc2497a0a40>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-4-23-trt4r8" in resourcegroup="rg-cp-ystream-upgrade-4-23-nm455m5dz4rg" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3",
      "name": "ab2c099b-e08d-48bf-8615-536073622db3",
      "status": "Failed",
      "startTime": "2026-10-14T03:17:19.784745543Z",
      "endTime": "2026-10-14T04:01:20.236478176Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab2c099b-e08d-48bf-8615-536073622db3
            correlation ID: b6e77c38-1227-46e6-9bc5-a4baba263145
            client request ID: fcf783cc-37c0-4ef5-8f68-49e5b907b57c
occurred
cluster <resource-group>/<cluster> entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
02:08:42 UTC · failed at e2e · test ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-jf2trk9s8chj/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.
cluster should eventually succeed after role assignments are created
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:
cluster delayed-rbac-jf2trk9s8chj/delayed-rbac-cluster entered terminal state "Failed" after role assignment deployment; expected "Succeeded"
Expected
    <armredhatopenshifthcp.ProvisioningState>: Failed
not to equal
    <armredhatopenshifthcp.ProvisioningState>: Failed
ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation "hypershift.openshift.io/cluster-size-override" is "e2e_minimal; provider Microsoft.RedHatOpenShift
02:08:42 UTC · failed at e2e · test Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-5-0-v9gdsd" to 5.0
Unexpected error:
    <*fmt.wrapError | 0x89c8a7b4520>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-5-0-v9gdsd" in resourcegroup="rg-cp-ystream-upgrade-5-0-lrtt9r887h59" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0",
      "name": "149ad663-a4d9-4ebd-9101-8c188eed65d0",
      "status": "Failed",
      "startTime": "2026-10-14T03:29:27.04612928Z",
      "endTime": "2026-10-14T04:13:34.922970159Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0
            correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7
            client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster "cp-ystream-upgrade-5-0-v9gdsd" to 5.0
Unexpected error:
    <*fmt.wrapError | 0x89c8a7b4520>: 
    failed waiting for hcpCluster="cp-ystream-upgrade-5-0-v9gdsd" in resourcegroup="rg-cp-ystream-upgrade-5-0-lrtt9r887h59" to finish updating: GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0",
      "name": "149ad663-a4d9-4ebd-9101-8c188eed65d0",
      "status": "Failed",
      "startTime": "2026-10-14T03:29:27.04612928Z",
      "endTime": "2026-10-14T04:13:34.922970159Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal\", want unset"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8816896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/149ad663-a4d9-4ebd-9101-8c188eed65d0
            correlation ID: b89bc015-38c4-4b8f-8cf6-fd7d29e794e7
            client request ID: c975ef93-0023-4a12-b6b6-8dfd3c0b6abf
occurred
2h 27m 50s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head eda29a9 · merge eda29a9
 centralus e2e failed ★#7510 (merged) 1
e2e (1)
failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T03:48:07
02:04:28 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.
Expected success, but got an error:
    <*fmt.wrapError | 0x1fce6bc96f40>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);
    let _endTime = datetime(2026-10-14T03:48:07.3491058Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-sbj59nsxlhq5'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T03:48:19.7777147Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958", ActivityId:"b81cad1f-eefa-4a7c-87bf-111e145d281b", SubActivityId:"b81cad1f-eefa-4a7c-87bf-111e145d281b", ActivityType:"GW.Http.CallContext", ParentActivityId:"b81cad1f-eefa-4a7c-87bf-111e145d281b", ActivityStack:"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x1fce6bb2e558>{
            errs: [
                <*fmt.wrapError | 0x1fce6bc96f20>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x1fce6bc96f00>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x1fce6b3e4d40>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x1fce6b3e4d00>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x1fce6d014600>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.
Expected success, but got an error:
    <*fmt.wrapError | 0x1fce6bc96f40>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);
    let _endTime = datetime(2026-10-14T03:48:07.3491058Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-sbj59nsxlhq5'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T03:48:19.7777147Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958", ActivityId:"b81cad1f-eefa-4a7c-87bf-111e145d281b", SubActivityId:"b81cad1f-eefa-4a7c-87bf-111e145d281b", ActivityType:"GW.Http.CallContext", ParentActivityId:"b81cad1f-eefa-4a7c-87bf-111e145d281b", ActivityStack:"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x1fce6bb2e558>{
            errs: [
                <*fmt.wrapError | 0x1fce6bc96f20>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x1fce6bc96f00>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x1fce6b3e4d40>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x1fce6b3e4d00>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T03:48:07.3491055Z);\nlet _endTime = datetime(2026-10-14T03:48:07.3491058Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-sbj59nsxlhq5'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T03:48:19.7777147Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958\", ActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", SubActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"b81cad1f-eefa-4a7c-87bf-111e145d281b\", ActivityStack:\"(Activity stack: CRID=KGC.execute;820fc79d-e119-44c9-84ae-f201c98c1958 ARID=b81cad1f-eefa-4a7c-87bf-111e145d281b > GW.Http.CallContext/b81cad1f-eefa-4a7c-87bf-111e145d281b)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x1fce6d014600>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
2h 22m 33s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head bb4f900 · merge bb4f900
 centralus e2e failed ★#7509 (merged) 1
e2e (1)
failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T02:13:05
00:39:37 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.
Expected success, but got an error:
    <*fmt.wrapError | 0x1e936bab0320>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);
    let _endTime = datetime(2026-10-14T02:13:05.3130233Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-zxm6njn97hd4'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T02:13:17.5036831Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6", ActivityId:"15fc144c-f310-411c-9ec4-c55a211606b5", SubActivityId:"15fc144c-f310-411c-9ec4-c55a211606b5", ActivityType:"GW.Http.CallContext", ParentActivityId:"15fc144c-f310-411c-9ec4-c55a211606b5", ActivityStack:"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x1e936b180db0>{
            errs: [
                <*fmt.wrapError | 0x1e936bab0300>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x1e936bab02e0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x1e93699ddd20>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x1e93699ddc60>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x1e936a2c1400>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.
Expected success, but got an error:
    <*fmt.wrapError | 0x1e936bab0320>: 
    failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query "detailedInfraOrchestrationLogs" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);
    let _endTime = datetime(2026-10-14T02:13:05.3130233Z);
    // Extract cluster ID from CS logs using cid field
    let _cs_cluster_id = toscalar(
        database('ServiceLogs').table('clustersServiceLogs')
        | where timestamp between (_startTime .. _endTime)
        | where resource_id has 'kusto-logs-zxm6njn97hd4'
        | where isnotempty(log.cid)
        | distinct tostring(log.cid)
        | limit 1
    );
    // Find all Maestro resource IDs for this cluster
    let maestro_resource_ids = (
        database('ServiceLogs').table('containerLogs')
        | where timestamp between (_startTime .. _endTime)
        | where namespace_name has "maestro"
        | extend logStr = tostring(log)
        | where logStr has _cs_cluster_id
        | extend resourceId = extract(@'resourceid[=:"]+([a-f0-9\-]+)', 1, logStr)
        | where isnotempty(resourceId)
        | distinct resourceId
    );
    // Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent
    union
        (
            // Maestro logs: Resource bundle operations for this cluster and its associated resources
            // Includes: resource creation, updates, deletions, status events, gRPC communications
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "maestro"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs
            // Includes the CR reconciliation logs
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "hypershift"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        ),
        (
            // ACM Agent logs: ManifestWork reconciliation on the managed cluster
            // Includes: work agent operations, manifest application, status feedback
            database('ServiceLogs').table('containerLogs')
            | where timestamp between (_startTime .. _endTime)
            | where namespace_name has "open-cluster-management-agent"
            | extend logStr = tostring(log)
            | where logStr has _cs_cluster_id
            | project timestamp, container_name, msg = logStr, log
        )
    | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:"LimitsExceeded", Message:"Request is invalid and cannot be executed.", Description:"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..", Type:"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException", Context:v2.ErrorContext{Timestamp:"2026-10-14T02:13:17.5036831Z", ServiceAlias:"HCP-DEV-US-2", MachineName:"", ProcessName:"", ProcessId:0, ThreadId:0, ClientRequestId:"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6", ActivityId:"15fc144c-f310-411c-9ec4-c55a211606b5", SubActivityId:"15fc144c-f310-411c-9ec4-c55a211606b5", ActivityType:"GW.Http.CallContext", ParentActivityId:"15fc144c-f310-411c-9ec4-c55a211606b5", ActivityStack:"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)"}, IsPermanent:false})
    {
        msg: "failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
        err: <*errors.joinError | 0x1e936b180db0>{
            errs: [
                <*fmt.wrapError | 0x1e936bab0300>{
                    msg: "failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                    err: <*fmt.wrapError | 0x1e936bab02e0>{
                        msg: "error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                        err: <*fmt.wrapError | 0x1e93699ddd20>{
                            msg: "failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                            err: <*fmt.wrapError | 0x1e93699ddc60>{
                                msg: "failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-13T02:13:05.3130232Z);\nlet _endTime = datetime(2026-10-14T02:13:05.3130233Z);\n// Extract cluster ID from CS logs using cid field\nlet _cs_cluster_id = toscalar(\n    database('ServiceLogs').table('clustersServiceLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where resource_id has 'kusto-logs-zxm6njn97hd4'\n    | where isnotempty(log.cid)\n    | distinct tostring(log.cid)\n    | limit 1\n);\n// Find all Maestro resource IDs for this cluster\nlet maestro_resource_ids = (\n    database('ServiceLogs').table('containerLogs')\n    | where timestamp between (_startTime .. _endTime)\n    | where namespace_name has \"maestro\"\n    | extend logStr = tostring(log)\n    | where logStr has _cs_cluster_id\n    | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n    | where isnotempty(resourceId)\n    | distinct resourceId\n);\n// Infrastructure & Orchestration Logs - Maestro, Hypershift, and ACM Agent\nunion\n    (\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"maestro\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n        // Includes the CR reconciliation logs\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"hypershift\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    ),\n    (\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n        // Includes: work agent operations, manifest application, status feedback\n        database('ServiceLogs').table('containerLogs')\n        | where timestamp between (_startTime .. _endTime)\n        | where namespace_name has \"open-cluster-management-agent\"\n        | extend logStr = tostring(log)\n        | where logStr has _cs_cluster_id\n        | project timestamp, container_name, msg = logStr, log\n    )\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-14T02:13:17.5036831Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6\", ActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", SubActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"15fc144c-f310-411c-9ec4-c55a211606b5\", ActivityStack:\"(Activity stack: CRID=KGC.execute;5d34b777-6393-4d20-9e49-e06cdb8937a6 ARID=15fc144c-f310-411c-9ec4-c55a211606b5 > GW.Http.CallContext/15fc144c-f310-411c-9ec4-c55a211606b5)\"}, IsPermanent:false})",
                                err: <*v2.OneApiError | 0x1e936a2c1400>{
                                    ErrorMessage: {
                                        Code: "LimitsExceeded",
                                        Message: "Request is invalid and cannot be executed.",
                                        Description: "Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..",
                                        Type: "Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException",
                                        Context: {
                                            Timestamp: ...,
                                            ServiceAlias: ...,
                                            MachineName: ...,
                                            ProcessName: ...,
                                            ProcessId: ...,
                                            ThreadId: ...,
                                            ClientRequestId: ...,
                                            ActivityId: ...,
                                            SubActivityId: ...,
                                            ActivityType: ...,
                                            ParentActivityId: ...,
                                            ActivityStack: ...,
                                        },
                                        IsPermanent: false,
                                    },
                                },
                            },
                        },
                    },
                },
            ],
        },
    }
2h 22m 35s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head e343803 · merge e343803
 westus3 e2e failed #7488 (merged) 1
e2e (1)
ERROR CODE: 429; provider Microsoft.ManagedIdentity
00:39:33 UTC · failed at e2e · test Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity
Unexpected error:
    <*exported.ResponseError | 0x18d22f431050>: 
    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure
    --------------------------------------------------------------------------------
    RESPONSE 429: 429 Too Many Requests
    ERROR CODE: 429
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "429",
        "message": "Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure
            correlation ID: d277ac90-384a-44a9-9799-7698621627f9
            client request ID: unavailable
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:119]: failed to get cluster-api-azure managed identity
Unexpected error:
    <*exported.ResponseError | 0x18d22f431050>: 
    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure
    --------------------------------------------------------------------------------
    RESPONSE 429: 429 Too Many Requests
    ERROR CODE: 429
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "429",
        "message": "Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure
            correlation ID: d277ac90-384a-44a9-9799-7698621627f9
            client request ID: unavailable
occurred
2h 19m 51s pull-ci-Azure-ARO-HCP-main-e2e-parallel
post-goodmerged PR
head 29ee7d9 · merge 29ee7d9
 centralus e2e failed ★#7248 (merged) 4
e2e (4)
ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is <count>, want <count>; node pool condition AllNodesHealthy is False: <count> of <count> machines are not healthy Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-<id> to exist Machine <node>: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; node pool condition AllMachinesReady is False: <count> of <count> machines are not ready Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-<id> to exist Machine <node>: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/<subscription>/resourcegroups/<resource-group>/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist; provider Microsoft.RedHatOpenShift
00:39:25 UTC · failed at e2e · test Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x147375bd60d8>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-l8lntszfbqhk" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a",
      "name": "1bccabc3-2126-4728-9d49-4eea01285f0a",
      "status": "Failed",
      "startTime": "2026-10-14T01:45:21.225915122Z",
      "endTime": "2026-10-14T02:14:22.961369975Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a
            correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e
            client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster "cilium-cluster"
Unexpected error:
    <*errors.joinError | 0x147375bd60d8>: 
    VerifyNodesReady failed: no nodes found in the cluster
    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded
    failed to create NodePool cilium-np: failed waiting for nodepool="cilium-np" for cluster "cilium-cluster" in resourcegroup="complex-cilium-kv-l8lntszfbqhk" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a",
      "name": "1bccabc3-2126-4728-9d49-4eea01285f0a",
      "status": "Failed",
      "startTime": "2026-10-14T01:45:21.225915122Z",
      "endTime": "2026-10-14T02:14:22.961369975Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine cilium-cluster-cilium-np-llxzk-7ntxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-7ntxq to exist\nMachine cilium-cluster-cilium-np-llxzk-gsvtb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-l8lntszfbqhk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-llxzk-gsvtb to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1bccabc3-2126-4728-9d49-4eea01285f0a
            correlation ID: 854376b3-85ce-463a-aabe-5fc52fac910e
            client request ID: d86cb5ff-fe28-499e-baef-054a0de770d9
occurred
failed to create nodepool <nodepool>: failed waiting for nodepool="<nodepool>" for cluster "<cluster>" in resourcegroup="<resource-group>" to finish creating: GET <url>
00:39:25 UTC · failed at e2e · test Customer should be able to delete a nodepool · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x3899558a41e0>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-znvkktb7cpf2" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4",
      "name": "61831d1e-3d9c-42ab-aa2d-123e54ce5ca4",
      "status": "Failed",
      "startTime": "2026-10-14T01:44:37.675473518Z",
      "endTime": "2026-10-14T02:13:43.0652181Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4
            correlation ID: 58d6d42f-a8fe-46ff-8c2a-c74cc76252de
            client request ID: ce3ec9eb-4220-47bf-877f-5472abdbec39
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x3899558a41e0>: 
    failed to create NodePool np-delete-main: failed waiting for nodepool="np-delete-main" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-znvkktb7cpf2" to finish creating: GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4",
      "name": "61831d1e-3d9c-42ab-aa2d-123e54ce5ca4",
      "status": "Failed",
      "startTime": "2026-10-14T01:44:37.675473518Z",
      "endTime": "2026-10-14T02:13:43.0652181Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-znvkktb7cpf2--managed/providers/Microsoft.Compute/virtualMachines/e5q1y3y5s1c7e2g-np-delete-main-7qtv8-2kdpd to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8586496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61831d1e-3d9c-42ab-aa2d-123e54ce5ca4
            correlation ID: 58d6d42f-a8fe-46ff-8c2a-c74cc76252de
            client request ID: ce3ec9eb-4220-47bf-877f-5472abdbec39
DNS for route host <route-host> did not resolve: DNS for <route-host> did not resolve within <duration> (last error: lookup <host> on <dns-server>: dial udp <ip>: i/o timeout): context deadline exceeded
00:39:25 UTC · failed at e2e · test ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22 · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-m9c8cf"
Unexpected error:
    <*fmt.wrapError | 0x27a2ab70e4c0>: 
    DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x27a2ab70e3e0>{
            msg: "DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-m9c8cf"
Unexpected error:
    <*fmt.wrapError | 0x27a2ab70e4c0>: 
    DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x27a2ab70e3e0>{
            msg: "DNS for agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2x889.apps.aro.y3a7f6t6b3g8g6t.a94j.j8586496.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
| take 101, Post "<url>": context deadline exceeded
00:39:25 UTC · failed at e2e · test Engineering should be able to retrieve kusto logs for a cluster and services · suite rp-api-compat-all/parallel
Show raw failure
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x25f598a00de0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T01:32:38.0694114Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x25f598a00dc0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x25f597dc0730>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x25f598a00da0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x25f59a111950>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x25f598a00de0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-14T01:32:38.0694114Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | join kind=leftsemi hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x25f598a00dc0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x25f597dc0730>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x25f598a00da0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cplxmksrs6q5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-14T01:32:38.0694114Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| join kind=leftsemi hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x25f59a111950>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred